Examples with TaskActionClient org.apache.druid.indexing.common.actions.TaskActionClient used on opensource projects

Example 16 with TaskActionClient

use of org.apache.druid.indexing.common.actions.TaskActionClient in project druid by druid-io.

the class ParallelIndexPhaseRunnerTest method testLargeEstimatedNumSplits.

@Test
public void testLargeEstimatedNumSplits() throws Exception {
    final NoopTask task = NoopTask.create();
    final TaskActionClient actionClient = createActionClient(task);
    final TaskToolbox toolbox = createTaskToolbox(task, actionClient);
    final TestPhaseRunner runner = new TestPhaseRunner(toolbox, "supervisorTaskId", "groupId", AbstractParallelIndexSupervisorTaskTest.DEFAULT_TUNING_CONFIG_FOR_PARALLEL_INDEXING, 10, 12);
    Assert.assertEquals(TaskState.SUCCESS, runner.run());
}

Example 17 with TaskActionClient

use of org.apache.druid.indexing.common.actions.TaskActionClient in project druid by druid-io.

the class SinglePhaseParallelIndexingTest method testIsReady.

@Test
public void testIsReady() throws Exception {
    final ParallelIndexSupervisorTask task = newTask(INTERVAL_TO_INDEX, false, true);
    final TaskActionClient actionClient = createActionClient(task);
    final TaskToolbox toolbox = createTaskToolbox(task, actionClient);
    prepareTaskForLocking(task);
    Assert.assertTrue(task.isReady(actionClient));
    final SinglePhaseParallelIndexTaskRunner runner = task.createSinglePhaseTaskRunner(toolbox);
    final Iterator<SubTaskSpec<SinglePhaseSubTask>> subTaskSpecIterator = runner.subTaskSpecIterator();
    while (subTaskSpecIterator.hasNext()) {
        final SinglePhaseSubTaskSpec spec = (SinglePhaseSubTaskSpec) subTaskSpecIterator.next();
        final SinglePhaseSubTask subTask = new SinglePhaseSubTask(null, spec.getGroupId(), null, spec.getSupervisorTaskId(), spec.getId(), 0, spec.getIngestionSpec(), spec.getContext());
        final TaskActionClient subTaskActionClient = createActionClient(subTask);
        prepareTaskForLocking(subTask);
        Assert.assertTrue(subTask.isReady(subTaskActionClient));
    }
}

Example 18 with TaskActionClient

use of org.apache.druid.indexing.common.actions.TaskActionClient in project druid by druid-io.

the class HadoopTaskTest method testBuildClassLoader.

@Test
public void testBuildClassLoader() throws Exception {
    final HadoopTask task = new HadoopTask("taskId", "dataSource", ImmutableList.of(), ImmutableMap.of()) {

        @Override
        public String getType() {
            return null;
        }

        @Override
        public boolean isReady(TaskActionClient taskActionClient) {
            return false;
        }

        @Override
        public void stopGracefully(TaskConfig taskConfig) {
        }

        @Override
        public boolean requireLockExistingSegments() {
            return true;
        }

        @Override
        public List<DataSegment> findSegmentsToLock(TaskActionClient taskActionClient, List<Interval> intervals) {
            return Collections.emptyList();
        }

        @Override
        public boolean isPerfectRollup() {
            return true;
        }

        @Nullable
        @Override
        public Granularity getSegmentGranularity() {
            return null;
        }

        @Override
        public TaskStatus runTask(TaskToolbox toolbox) {
            return null;
        }
    };
    final TaskToolbox toolbox = EasyMock.createStrictMock(TaskToolbox.class);
    EasyMock.expect(toolbox.getConfig()).andReturn(new TaskConfig(temporaryFolder.newFolder().toString(), null, null, null, ImmutableList.of("something:hadoop:1"), false, null, null, null, false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name())).once();
    EasyMock.replay(toolbox);
    final ClassLoader classLoader = task.buildClassLoader(toolbox);
    assertClassLoaderIsSingular(classLoader);
    final Class<?> hadoopClazz = Class.forName("org.apache.hadoop.fs.FSDataInputStream", false, classLoader);
    assertClassLoaderIsSingular(hadoopClazz.getClassLoader());
    final Class<?> druidHadoopConfigClazz = Class.forName("org.apache.druid.indexer.HadoopDruidIndexerConfig", false, classLoader);
    assertClassLoaderIsSingular(druidHadoopConfigClazz.getClassLoader());
}

Example 19 with TaskActionClient

use of org.apache.druid.indexing.common.actions.TaskActionClient in project druid by druid-io.

the class AbstractBatchIndexTask method findInputSegments.

/**
 * If the given firehoseFactory is {@link IngestSegmentFirehoseFactory}, then it finds the segments to lock
 * from the firehoseFactory. This is because those segments will be read by this task no matter what segments would be
 * filtered by intervalsToRead, so they need to be locked.
 * <p>
 * However, firehoseFactory is not IngestSegmentFirehoseFactory, it means this task will overwrite some segments
 * with data read from some input source outside of Druid. As a result, only the segments falling in intervalsToRead
 * should be locked.
 * <p>
 * The order of segments within the returned list is unspecified, but each segment is guaranteed to appear in the list
 * only once.
 */
protected static List<DataSegment> findInputSegments(String dataSource, TaskActionClient actionClient, List<Interval> intervalsToRead, FirehoseFactory firehoseFactory) throws IOException {
    if (firehoseFactory instanceof IngestSegmentFirehoseFactory) {
        // intervalsToRead is ignored here.
        final List<WindowedSegmentId> inputSegments = ((IngestSegmentFirehoseFactory) firehoseFactory).getSegments();
        if (inputSegments == null) {
            final Interval inputInterval = Preconditions.checkNotNull(((IngestSegmentFirehoseFactory) firehoseFactory).getInterval(), "input interval");
            return ImmutableList.copyOf(actionClient.submit(new RetrieveUsedSegmentsAction(dataSource, inputInterval, null, Segments.ONLY_VISIBLE)));
        } else {
            final List<String> inputSegmentIds = inputSegments.stream().map(WindowedSegmentId::getSegmentId).collect(Collectors.toList());
            final Collection<DataSegment> dataSegmentsInIntervals = actionClient.submit(new RetrieveUsedSegmentsAction(dataSource, null, inputSegments.stream().flatMap(windowedSegmentId -> windowedSegmentId.getIntervals().stream()).collect(Collectors.toSet()), Segments.ONLY_VISIBLE));
            return dataSegmentsInIntervals.stream().filter(segment -> inputSegmentIds.contains(segment.getId().toString())).collect(Collectors.toList());
        }
    } else {
        return ImmutableList.copyOf(actionClient.submit(new RetrieveUsedSegmentsAction(dataSource, null, intervalsToRead, Segments.ONLY_VISIBLE)));
    }
}

Example 20 with TaskActionClient

use of org.apache.druid.indexing.common.actions.TaskActionClient in project druid by druid-io.

the class WorkerTaskManagerTest method createWorkerTaskManager.

private WorkerTaskManager createWorkerTaskManager() {
    TaskConfig taskConfig = new TaskConfig(FileUtils.createTempDir().toString(), null, null, 0, null, false, null, null, null, false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name());
    TaskActionClientFactory taskActionClientFactory = EasyMock.createNiceMock(TaskActionClientFactory.class);
    TaskActionClient taskActionClient = EasyMock.createNiceMock(TaskActionClient.class);
    EasyMock.expect(taskActionClientFactory.create(EasyMock.anyObject())).andReturn(taskActionClient).anyTimes();
    SegmentHandoffNotifierFactory notifierFactory = EasyMock.createNiceMock(SegmentHandoffNotifierFactory.class);
    EasyMock.replay(taskActionClientFactory, taskActionClient, notifierFactory);
    return new WorkerTaskManager(jsonMapper, new TestTaskRunner(new TaskToolboxFactory(taskConfig, null, taskActionClientFactory, null, null, null, null, null, null, null, notifierFactory, null, null, NoopJoinableFactory.INSTANCE, null, new SegmentCacheManagerFactory(jsonMapper), jsonMapper, indexIO, null, null, null, indexMergerV9, null, null, null, null, new NoopTestTaskReportFileWriter(), null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, new NoopChatHandlerProvider(), testUtils.getRowIngestionMetersFactory(), new TestAppenderatorsManager(), new NoopIndexingServiceClient(), null, null, null), taskConfig, location), taskConfig, EasyMock.createNiceMock(DruidLeaderClient.class)) {

        @Override
        protected void taskStarted(String taskId) {
        }

        @Override
        protected void taskAnnouncementChanged(TaskAnnouncement announcement) {
        }
    };
}