Examples with TaskConfig org.apache.druid.indexing.common.config.TaskConfig used on opensource projects

Example 6 with TaskConfig

use of org.apache.druid.indexing.common.config.TaskConfig in project druid by druid-io.

the class ThreadingTaskRunnerTest method testTaskStatusWhenTaskThrowsExceptionWhileRunning.

@Test
public void testTaskStatusWhenTaskThrowsExceptionWhileRunning() throws ExecutionException, InterruptedException {
    ThreadingTaskRunner runner = new ThreadingTaskRunner(mockTaskToolboxFactory(), new TaskConfig(null, null, null, null, ImmutableList.of(), false, new Period("PT0S"), new Period("PT10S"), ImmutableList.of(), false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name()), new WorkerConfig(), new NoopTaskLogs(), new DefaultObjectMapper(), new TestAppenderatorsManager(), new MultipleFileTaskReportFileWriter(), new DruidNode("middleManager", "host", false, 8091, null, true, false));
    Future<TaskStatus> statusFuture = runner.run(new AbstractTask("id", "datasource", null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public boolean isReady(TaskActionClient taskActionClient) {
            return true;
        }

        @Override
        public void stopGracefully(TaskConfig taskConfig) {
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) {
            throw new RuntimeException("Task failure test");
        }
    });
    TaskStatus status = statusFuture.get();
    Assert.assertEquals(TaskState.FAILED, status.getStatusCode());
    Assert.assertEquals("Failed with an exception. See indexer logs for more details.", status.getErrorMsg());
}

Example 7 with TaskConfig

use of org.apache.druid.indexing.common.config.TaskConfig in project druid by druid-io.

the class ForkingTaskRunnerTest method testTaskStatusWhenTaskProcessSucceedsTaskFails.

@Test
public void testTaskStatusWhenTaskProcessSucceedsTaskFails() throws ExecutionException, InterruptedException {
    ObjectMapper mapper = new DefaultObjectMapper();
    Task task = NoopTask.create();
    ForkingTaskRunner forkingTaskRunner = new ForkingTaskRunner(new ForkingTaskRunnerConfig(), new TaskConfig(null, null, null, null, ImmutableList.of(), false, new Period("PT0S"), new Period("PT10S"), ImmutableList.of(), false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name()), new WorkerConfig(), new Properties(), new NoopTaskLogs(), mapper, new DruidNode("middleManager", "host", false, 8091, null, true, false), new StartupLoggingConfig()) {

        @Override
        ProcessHolder runTaskProcess(List<String> command, File logFile, TaskLocation taskLocation) throws IOException {
            ProcessHolder processHolder = Mockito.mock(ProcessHolder.class);
            Mockito.doNothing().when(processHolder).registerWithCloser(ArgumentMatchers.any());
            Mockito.doNothing().when(processHolder).shutdown();
            for (String param : command) {
                if (param.endsWith("status.json")) {
                    mapper.writeValue(new File(param), TaskStatus.failure(task.getId(), "task failure test"));
                    break;
                }
            }
            return processHolder;
        }

        @Override
        int waitForTaskProcessToComplete(Task task, ProcessHolder processHolder, File logFile, File reportsFile) {
            return 0;
        }
    };
    final TaskStatus status = forkingTaskRunner.run(task).get();
    Assert.assertEquals(TaskState.FAILED, status.getStatusCode());
    Assert.assertEquals("task failure test", status.getErrorMsg());
}

Example 8 with TaskConfig

use of org.apache.druid.indexing.common.config.TaskConfig in project druid by druid-io.

the class ForkingTaskRunnerTest method testTaskStatusWhenTaskProcessSucceedsTaskSucceeds.

@Test
public void testTaskStatusWhenTaskProcessSucceedsTaskSucceeds() throws ExecutionException, InterruptedException {
    ObjectMapper mapper = new DefaultObjectMapper();
    Task task = NoopTask.create();
    ForkingTaskRunner forkingTaskRunner = new ForkingTaskRunner(new ForkingTaskRunnerConfig(), new TaskConfig(null, null, null, null, ImmutableList.of(), false, new Period("PT0S"), new Period("PT10S"), ImmutableList.of(), false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name()), new WorkerConfig(), new Properties(), new NoopTaskLogs(), mapper, new DruidNode("middleManager", "host", false, 8091, null, true, false), new StartupLoggingConfig()) {

        @Override
        ProcessHolder runTaskProcess(List<String> command, File logFile, TaskLocation taskLocation) throws IOException {
            ProcessHolder processHolder = Mockito.mock(ProcessHolder.class);
            Mockito.doNothing().when(processHolder).registerWithCloser(ArgumentMatchers.any());
            Mockito.doNothing().when(processHolder).shutdown();
            for (String param : command) {
                if (param.endsWith("status.json")) {
                    mapper.writeValue(new File(param), TaskStatus.success(task.getId()));
                    break;
                }
            }
            return processHolder;
        }

        @Override
        int waitForTaskProcessToComplete(Task task, ProcessHolder processHolder, File logFile, File reportsFile) {
            return 0;
        }
    };
    final TaskStatus status = forkingTaskRunner.run(task).get();
    Assert.assertEquals(TaskState.SUCCESS, status.getStatusCode());
    Assert.assertNull(status.getErrorMsg());
}

Example 9 with TaskConfig

use of org.apache.druid.indexing.common.config.TaskConfig in project druid by druid-io.

the class WorkerTaskMonitorTest method createTaskMonitor.

private WorkerTaskMonitor createTaskMonitor() {
    final TaskConfig taskConfig = new TaskConfig(FileUtils.createTempDir().toString(), null, null, 0, null, false, null, null, null, false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name());
    TaskActionClientFactory taskActionClientFactory = EasyMock.createNiceMock(TaskActionClientFactory.class);
    TaskActionClient taskActionClient = EasyMock.createNiceMock(TaskActionClient.class);
    EasyMock.expect(taskActionClientFactory.create(EasyMock.anyObject())).andReturn(taskActionClient).anyTimes();
    SegmentHandoffNotifierFactory notifierFactory = EasyMock.createNiceMock(SegmentHandoffNotifierFactory.class);
    EasyMock.replay(taskActionClientFactory, taskActionClient, notifierFactory);
    return new WorkerTaskMonitor(jsonMapper, new SingleTaskBackgroundRunner(new TaskToolboxFactory(taskConfig, null, taskActionClientFactory, null, null, null, null, null, null, null, notifierFactory, null, null, NoopJoinableFactory.INSTANCE, null, new SegmentCacheManagerFactory(jsonMapper), jsonMapper, indexIO, null, null, null, indexMergerV9, null, null, null, null, new NoopTestTaskReportFileWriter(), null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, new NoopChatHandlerProvider(), testUtils.getRowIngestionMetersFactory(), new TestAppenderatorsManager(), new NoopIndexingServiceClient(), null, null, null), taskConfig, new NoopServiceEmitter(), DUMMY_NODE, new ServerConfig()), taskConfig, cf, workerCuratorCoordinator, EasyMock.createNiceMock(DruidLeaderClient.class));
}

Example 10 with TaskConfig

use of org.apache.druid.indexing.common.config.TaskConfig in project druid by druid-io.

the class KinesisIndexTaskTest method makeToolboxFactory.

private void makeToolboxFactory() throws IOException {
    directory = tempFolder.newFolder();
    final TestUtils testUtils = new TestUtils();
    final ObjectMapper objectMapper = testUtils.getTestObjectMapper();
    objectMapper.setInjectableValues(((InjectableValues.Std) objectMapper.getInjectableValues()).addValue(AWSCredentialsConfig.class, new AWSCredentialsConfig()));
    for (Module module : new KinesisIndexingServiceModule().getJacksonModules()) {
        objectMapper.registerModule(module);
    }
    final TaskConfig taskConfig = new TaskConfig(new File(directory, "baseDir").getPath(), new File(directory, "baseTaskDir").getPath(), null, 50000, null, true, null, null, null, false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name());
    final TestDerbyConnector derbyConnector = derby.getConnector();
    derbyConnector.createDataSourceTable();
    derbyConnector.createPendingSegmentsTable();
    derbyConnector.createSegmentTable();
    derbyConnector.createRulesTable();
    derbyConnector.createConfigTable();
    derbyConnector.createTaskTables();
    derbyConnector.createAuditTable();
    taskStorage = new MetadataTaskStorage(derbyConnector, new TaskStorageConfig(null), new DerbyMetadataStorageActionHandlerFactory(derbyConnector, derby.metadataTablesConfigSupplier().get(), objectMapper));
    metadataStorageCoordinator = new IndexerSQLMetadataStorageCoordinator(testUtils.getTestObjectMapper(), derby.metadataTablesConfigSupplier().get(), derbyConnector);
    taskLockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator);
    final TaskActionToolbox taskActionToolbox = new TaskActionToolbox(taskLockbox, taskStorage, metadataStorageCoordinator, emitter, new SupervisorManager(null) {

        @Override
        public boolean checkPointDataSourceMetadata(String supervisorId, int taskGroupId, @Nullable DataSourceMetadata checkpointMetadata) {
            LOG.info("Adding checkpoint hash to the set");
            checkpointRequestsHash.add(Objects.hash(supervisorId, taskGroupId, checkpointMetadata));
            return true;
        }
    });
    final TaskActionClientFactory taskActionClientFactory = new LocalTaskActionClientFactory(taskStorage, taskActionToolbox, new TaskAuditLogConfig(false));
    final SegmentHandoffNotifierFactory handoffNotifierFactory = dataSource -> new SegmentHandoffNotifier() {

        @Override
        public boolean registerSegmentHandoffCallback(SegmentDescriptor descriptor, Executor exec, Runnable handOffRunnable) {
            if (doHandoff) {
                // Simulate immediate handoff
                exec.execute(handOffRunnable);
            }
            return true;
        }

        @Override
        public void start() {
        // Noop
        }

        @Override
        public void close() {
        // Noop
        }
    };
    final LocalDataSegmentPusherConfig dataSegmentPusherConfig = new LocalDataSegmentPusherConfig();
    dataSegmentPusherConfig.storageDirectory = getSegmentDirectory();
    final DataSegmentPusher dataSegmentPusher = new LocalDataSegmentPusher(dataSegmentPusherConfig);
    toolboxFactory = new TaskToolboxFactory(taskConfig, // taskExecutorNode
    null, taskActionClientFactory, emitter, dataSegmentPusher, new TestDataSegmentKiller(), // DataSegmentMover
    null, // DataSegmentArchiver
    null, new TestDataSegmentAnnouncer(), EasyMock.createNiceMock(DataSegmentServerAnnouncer.class), handoffNotifierFactory, this::makeTimeseriesOnlyConglomerate, DirectQueryProcessingPool.INSTANCE, NoopJoinableFactory.INSTANCE, () -> EasyMock.createMock(MonitorScheduler.class), new SegmentCacheManagerFactory(testUtils.getTestObjectMapper()), testUtils.getTestObjectMapper(), testUtils.getTestIndexIO(), MapCache.create(1024), new CacheConfig(), new CachePopulatorStats(), testUtils.getTestIndexMergerV9(), EasyMock.createNiceMock(DruidNodeAnnouncer.class), EasyMock.createNiceMock(DruidNode.class), new LookupNodeService("tier"), new DataNodeService("tier", 1, ServerType.INDEXER_EXECUTOR, 0), new SingleFileTaskReportFileWriter(reportsFile), null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, new NoopChatHandlerProvider(), testUtils.getRowIngestionMetersFactory(), new TestAppenderatorsManager(), new NoopIndexingServiceClient(), null, null, null);
}