Examples with WorkerConfig org.apache.druid.indexing.worker.config.WorkerConfig used on opensource projects

Example 1 with WorkerConfig

use of org.apache.druid.indexing.worker.config.WorkerConfig in project druid by druid-io.

the class AbstractParallelIndexSupervisorTaskTest method setUpAbstractParallelIndexSupervisorTaskTest.

@Before
public void setUpAbstractParallelIndexSupervisorTaskTest() throws IOException {
    localDeepStorage = temporaryFolder.newFolder("localStorage");
    taskRunner = new SimpleThreadingTaskRunner();
    objectMapper = getObjectMapper();
    indexingServiceClient = new LocalIndexingServiceClient(objectMapper, taskRunner);
    intermediaryDataManager = new LocalIntermediaryDataManager(new WorkerConfig(), new TaskConfig(null, null, null, null, null, false, null, null, ImmutableList.of(new StorageLocationConfig(temporaryFolder.newFolder(), null, null)), false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name()), null);
    remoteApiExecutor = Execs.singleThreaded("coordinator-api-executor");
    coordinatorClient = new LocalCoordinatorClient(remoteApiExecutor);
    prepareObjectMapper(objectMapper, getIndexIO());
}

Example 2 with WorkerConfig

use of org.apache.druid.indexing.worker.config.WorkerConfig in project druid by druid-io.

the class TaskLifecycleTest method testUnifiedAppenderatorsManagerCleanup.

@Test
public void testUnifiedAppenderatorsManagerCleanup() throws Exception {
    final ExecutorService exec = Executors.newFixedThreadPool(8);
    UnifiedIndexerAppenderatorsManager unifiedIndexerAppenderatorsManager = new UnifiedIndexerAppenderatorsManager(new ForwardingQueryProcessingPool(exec), NoopJoinableFactory.INSTANCE, new WorkerConfig(), MapCache.create(2048), new CacheConfig(), new CachePopulatorStats(), MAPPER, new NoopServiceEmitter(), () -> queryRunnerFactoryConglomerate);
    tb = setUpTaskToolboxFactory(dataSegmentPusher, handoffNotifierFactory, mdc, unifiedIndexerAppenderatorsManager);
    taskRunner = setUpThreadPoolTaskRunner(tb);
    taskQueue = setUpTaskQueue(taskStorage, taskRunner);
    final Task indexTask = new IndexTask(null, null, new IndexIngestionSpec(new DataSchema("foo", new TimestampSpec(null, null, null), DimensionsSpec.EMPTY, new AggregatorFactory[] { new DoubleSumAggregatorFactory("met", "met") }, new UniformGranularitySpec(Granularities.DAY, null, ImmutableList.of(Intervals.of("2010-01-01/P2D"))), null), new IndexIOConfig(null, new MockInputSource(), new NoopInputFormat(), false, false), new IndexTuningConfig(null, 10000, null, 10, null, null, null, null, null, null, null, indexSpec, null, 3, false, null, null, null, null, null, null, null, null, null)), null);
    final Optional<TaskStatus> preRunTaskStatus = tsqa.getStatus(indexTask.getId());
    Assert.assertTrue("pre run task status not present", !preRunTaskStatus.isPresent());
    final TaskStatus mergedStatus = runTask(indexTask);
    final TaskStatus status = taskStorage.getStatus(indexTask.getId()).get();
    Assert.assertEquals("statusCode", TaskState.SUCCESS, status.getStatusCode());
    Map<String, UnifiedIndexerAppenderatorsManager.DatasourceBundle> bundleMap = unifiedIndexerAppenderatorsManager.getDatasourceBundles();
    Assert.assertEquals(1, bundleMap.size());
    unifiedIndexerAppenderatorsManager.removeAppenderatorsForTask(indexTask.getId(), "foo");
    Assert.assertTrue(bundleMap.isEmpty());
}

Example 3 with WorkerConfig

use of org.apache.druid.indexing.worker.config.WorkerConfig in project druid by druid-io.

the class ThreadingTaskRunnerTest method testTaskStatusWhenTaskThrowsExceptionWhileRunning.

@Test
public void testTaskStatusWhenTaskThrowsExceptionWhileRunning() throws ExecutionException, InterruptedException {
    ThreadingTaskRunner runner = new ThreadingTaskRunner(mockTaskToolboxFactory(), new TaskConfig(null, null, null, null, ImmutableList.of(), false, new Period("PT0S"), new Period("PT10S"), ImmutableList.of(), false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name()), new WorkerConfig(), new NoopTaskLogs(), new DefaultObjectMapper(), new TestAppenderatorsManager(), new MultipleFileTaskReportFileWriter(), new DruidNode("middleManager", "host", false, 8091, null, true, false));
    Future<TaskStatus> statusFuture = runner.run(new AbstractTask("id", "datasource", null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public boolean isReady(TaskActionClient taskActionClient) {
            return true;
        }

        @Override
        public void stopGracefully(TaskConfig taskConfig) {
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) {
            throw new RuntimeException("Task failure test");
        }
    });
    TaskStatus status = statusFuture.get();
    Assert.assertEquals(TaskState.FAILED, status.getStatusCode());
    Assert.assertEquals("Failed with an exception. See indexer logs for more details.", status.getErrorMsg());
}

Example 4 with WorkerConfig

use of org.apache.druid.indexing.worker.config.WorkerConfig in project druid by druid-io.

the class ForkingTaskRunnerTest method testTaskStatusWhenTaskProcessSucceedsTaskFails.

@Test
public void testTaskStatusWhenTaskProcessSucceedsTaskFails() throws ExecutionException, InterruptedException {
    ObjectMapper mapper = new DefaultObjectMapper();
    Task task = NoopTask.create();
    ForkingTaskRunner forkingTaskRunner = new ForkingTaskRunner(new ForkingTaskRunnerConfig(), new TaskConfig(null, null, null, null, ImmutableList.of(), false, new Period("PT0S"), new Period("PT10S"), ImmutableList.of(), false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name()), new WorkerConfig(), new Properties(), new NoopTaskLogs(), mapper, new DruidNode("middleManager", "host", false, 8091, null, true, false), new StartupLoggingConfig()) {

        @Override
        ProcessHolder runTaskProcess(List<String> command, File logFile, TaskLocation taskLocation) throws IOException {
            ProcessHolder processHolder = Mockito.mock(ProcessHolder.class);
            Mockito.doNothing().when(processHolder).registerWithCloser(ArgumentMatchers.any());
            Mockito.doNothing().when(processHolder).shutdown();
            for (String param : command) {
                if (param.endsWith("status.json")) {
                    mapper.writeValue(new File(param), TaskStatus.failure(task.getId(), "task failure test"));
                    break;
                }
            }
            return processHolder;
        }

        @Override
        int waitForTaskProcessToComplete(Task task, ProcessHolder processHolder, File logFile, File reportsFile) {
            return 0;
        }
    };
    final TaskStatus status = forkingTaskRunner.run(task).get();
    Assert.assertEquals(TaskState.FAILED, status.getStatusCode());
    Assert.assertEquals("task failure test", status.getErrorMsg());
}

Example 5 with WorkerConfig

use of org.apache.druid.indexing.worker.config.WorkerConfig in project druid by druid-io.

the class ForkingTaskRunnerTest method testTaskStatusWhenTaskProcessSucceedsTaskSucceeds.

@Test
public void testTaskStatusWhenTaskProcessSucceedsTaskSucceeds() throws ExecutionException, InterruptedException {
    ObjectMapper mapper = new DefaultObjectMapper();
    Task task = NoopTask.create();
    ForkingTaskRunner forkingTaskRunner = new ForkingTaskRunner(new ForkingTaskRunnerConfig(), new TaskConfig(null, null, null, null, ImmutableList.of(), false, new Period("PT0S"), new Period("PT10S"), ImmutableList.of(), false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name()), new WorkerConfig(), new Properties(), new NoopTaskLogs(), mapper, new DruidNode("middleManager", "host", false, 8091, null, true, false), new StartupLoggingConfig()) {

        @Override
        ProcessHolder runTaskProcess(List<String> command, File logFile, TaskLocation taskLocation) throws IOException {
            ProcessHolder processHolder = Mockito.mock(ProcessHolder.class);
            Mockito.doNothing().when(processHolder).registerWithCloser(ArgumentMatchers.any());
            Mockito.doNothing().when(processHolder).shutdown();
            for (String param : command) {
                if (param.endsWith("status.json")) {
                    mapper.writeValue(new File(param), TaskStatus.success(task.getId()));
                    break;
                }
            }
            return processHolder;
        }

        @Override
        int waitForTaskProcessToComplete(Task task, ProcessHolder processHolder, File logFile, File reportsFile) {
            return 0;
        }
    };
    final TaskStatus status = forkingTaskRunner.run(task).get();
    Assert.assertEquals(TaskState.SUCCESS, status.getStatusCode());
    Assert.assertNull(status.getErrorMsg());
}