Examples with TaskQueue org.apache.druid.indexing.overlord.TaskQueue used on opensource projects

Example 6 with TaskQueue

use of org.apache.druid.indexing.overlord.TaskQueue in project druid by druid-io.

the class KinesisSupervisorTest method setupTest.

@Before
public void setupTest() {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClient = createMock(KinesisIndexTaskClient.class);
    taskQueue = createMock(TaskQueue.class);
    supervisorRecordSupplier = createMock(KinesisRecordSupplier.class);
    tuningConfig = new KinesisSupervisorTuningConfig(null, 1000, null, null, 50000, null, new Period("P1Y"), new File("/test"), null, null, null, false, null, null, null, null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, TEST_HTTP_TIMEOUT, TEST_SHUTDOWN_TIMEOUT, null, null, null, 5000, null, null, null, null, null, null, null, null, null);
    rowIngestionMetersFactory = new TestUtils().getRowIngestionMetersFactory();
    serviceEmitter = new ExceptionCapturingServiceEmitter();
    EmittingLogger.registerEmitter(serviceEmitter);
    supervisorConfig = new SupervisorStateManagerConfig();
}

Example 7 with TaskQueue

use of org.apache.druid.indexing.overlord.TaskQueue in project druid by druid-io.

the class SeekableStreamSupervisor method createTasksForGroup.

private void createTasksForGroup(int groupId, int replicas) throws JsonProcessingException {
    TaskGroup group = activelyReadingTaskGroups.get(groupId);
    Map<PartitionIdType, SequenceOffsetType> startPartitions = group.startingSequences;
    Map<PartitionIdType, SequenceOffsetType> endPartitions = new HashMap<>();
    for (PartitionIdType partition : startPartitions.keySet()) {
        endPartitions.put(partition, getEndOfPartitionMarker());
    }
    Set<PartitionIdType> exclusiveStartSequenceNumberPartitions = activelyReadingTaskGroups.get(groupId).exclusiveStartSequenceNumberPartitions;
    DateTime minimumMessageTime = group.minimumMessageTime.orNull();
    DateTime maximumMessageTime = group.maximumMessageTime.orNull();
    SeekableStreamIndexTaskIOConfig newIoConfig = createTaskIoConfig(groupId, startPartitions, endPartitions, group.baseSequenceName, minimumMessageTime, maximumMessageTime, exclusiveStartSequenceNumberPartitions, ioConfig);
    List<SeekableStreamIndexTask<PartitionIdType, SequenceOffsetType, RecordType>> taskList = createIndexTasks(replicas, group.baseSequenceName, sortingMapper, group.checkpointSequences, newIoConfig, taskTuningConfig, rowIngestionMetersFactory);
    for (SeekableStreamIndexTask indexTask : taskList) {
        Optional<TaskQueue> taskQueue = taskMaster.getTaskQueue();
        if (taskQueue.isPresent()) {
            try {
                taskQueue.get().add(indexTask);
            } catch (EntryExistsException e) {
                stateManager.recordThrowableEvent(e);
                log.error("Tried to add task [%s] but it already exists", indexTask.getId());
            }
        } else {
            log.error("Failed to get task queue because I'm not the leader!");
        }
    }
}

Example 8 with TaskQueue

use of org.apache.druid.indexing.overlord.TaskQueue in project druid by druid-io.

the class SeekableStreamSupervisorStateTest method setupTest.

@Before
public void setupTest() {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    taskQueue = createMock(TaskQueue.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClientFactory = createMock(SeekableStreamIndexTaskClientFactory.class);
    spec = createMock(SeekableStreamSupervisorSpec.class);
    indexTaskClient = createMock(SeekableStreamIndexTaskClient.class);
    recordSupplier = (RecordSupplier<String, String, ByteEntity>) createMock(RecordSupplier.class);
    rowIngestionMetersFactory = new TestUtils().getRowIngestionMetersFactory();
    supervisorConfig = new SupervisorStateManagerConfig();
    emitter = new TestEmitter();
    EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes();
    EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes();
    EasyMock.expect(spec.getIoConfig()).andReturn(getIOConfig()).anyTimes();
    EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes();
    EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes();
    EasyMock.expect(taskClientFactory.build(EasyMock.anyObject(), EasyMock.anyString(), EasyMock.anyInt(), EasyMock.anyObject(), EasyMock.anyLong())).andReturn(indexTaskClient).anyTimes();
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
    taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class));
    EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(null).anyTimes();
    EasyMock.expect(recordSupplier.getAssignment()).andReturn(ImmutableSet.of(SHARD0_PARTITION)).anyTimes();
    EasyMock.expect(recordSupplier.getLatestSequenceNumber(EasyMock.anyObject())).andReturn("10").anyTimes();
}

Example 9 with TaskQueue

use of org.apache.druid.indexing.overlord.TaskQueue in project druid by druid-io.

the class OverlordResourceTest method testShutdownAllTasks.

@Test
public void testShutdownAllTasks() {
    // This is disabled since OverlordResource.shutdownTasksForDataSource is annotated with DatasourceResourceFilter
    // This should be fixed in https://github.com/apache/druid/issues/6685.
    // expectAuthorizationTokenCheck();
    TaskQueue mockQueue = EasyMock.createMock(TaskQueue.class);
    EasyMock.expect(taskMaster.isLeader()).andReturn(true).anyTimes();
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(mockQueue)).anyTimes();
    EasyMock.expect(taskStorageQueryAdapter.getActiveTaskInfo("datasource")).andStubReturn(ImmutableList.of(new TaskInfo("id_1", DateTime.now(ISOChronology.getInstanceUTC()), TaskStatus.success("id_1"), "datasource", getTaskWithIdAndDatasource("id_1", "datasource")), new TaskInfo("id_2", DateTime.now(ISOChronology.getInstanceUTC()), TaskStatus.success("id_2"), "datasource", getTaskWithIdAndDatasource("id_2", "datasource"))));
    mockQueue.shutdown("id_1", "Shutdown request from user");
    EasyMock.expectLastCall();
    mockQueue.shutdown("id_2", "Shutdown request from user");
    EasyMock.expectLastCall();
    EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req, mockQueue, workerTaskRunnerQueryAdapter);
    final Map<String, Integer> response = (Map<String, Integer>) overlordResource.shutdownTasksForDataSource("datasource").getEntity();
    Assert.assertEquals("datasource", response.get("dataSource"));
}