Examples with SeekableStreamIndexTaskIOConfig org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskIOConfig used on opensource projects

Example 1 with SeekableStreamIndexTaskIOConfig

use of org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskIOConfig in project druid by druid-io.

the class SeekableStreamSupervisor method createTasksForGroup.

private void createTasksForGroup(int groupId, int replicas) throws JsonProcessingException {
    TaskGroup group = activelyReadingTaskGroups.get(groupId);
    Map<PartitionIdType, SequenceOffsetType> startPartitions = group.startingSequences;
    Map<PartitionIdType, SequenceOffsetType> endPartitions = new HashMap<>();
    for (PartitionIdType partition : startPartitions.keySet()) {
        endPartitions.put(partition, getEndOfPartitionMarker());
    }
    Set<PartitionIdType> exclusiveStartSequenceNumberPartitions = activelyReadingTaskGroups.get(groupId).exclusiveStartSequenceNumberPartitions;
    DateTime minimumMessageTime = group.minimumMessageTime.orNull();
    DateTime maximumMessageTime = group.maximumMessageTime.orNull();
    SeekableStreamIndexTaskIOConfig newIoConfig = createTaskIoConfig(groupId, startPartitions, endPartitions, group.baseSequenceName, minimumMessageTime, maximumMessageTime, exclusiveStartSequenceNumberPartitions, ioConfig);
    List<SeekableStreamIndexTask<PartitionIdType, SequenceOffsetType, RecordType>> taskList = createIndexTasks(replicas, group.baseSequenceName, sortingMapper, group.checkpointSequences, newIoConfig, taskTuningConfig, rowIngestionMetersFactory);
    for (SeekableStreamIndexTask indexTask : taskList) {
        Optional<TaskQueue> taskQueue = taskMaster.getTaskQueue();
        if (taskQueue.isPresent()) {
            try {
                taskQueue.get().add(indexTask);
            } catch (EntryExistsException e) {
                stateManager.recordThrowableEvent(e);
                log.error("Tried to add task [%s] but it already exists", indexTask.getId());
            }
        } else {
            log.error("Failed to get task queue because I'm not the leader!");
        }
    }
}