Examples with KafkaIndexTaskTuningConfig org.apache.druid.indexing.kafka.KafkaIndexTaskTuningConfig used on opensource projects

Example 1 with KafkaIndexTaskTuningConfig

use of org.apache.druid.indexing.kafka.KafkaIndexTaskTuningConfig in project druid by druid-io.

the class KafkaSupervisorTest method testCreateBaseTaskContexts.

@Test
public void testCreateBaseTaskContexts() throws JsonProcessingException {
    supervisor = getTestableSupervisor(1, 1, true, "PT1H", null, null);
    final Map<String, Object> contexts = supervisor.createIndexTasks(1, "seq", OBJECT_MAPPER, new TreeMap<>(), new KafkaIndexTaskIOConfig(0, "seq", new SeekableStreamStartSequenceNumbers<>("test", Collections.emptyMap(), Collections.emptySet()), new SeekableStreamEndSequenceNumbers<>("test", Collections.emptyMap()), Collections.emptyMap(), null, null, null, null, INPUT_FORMAT), new KafkaIndexTaskTuningConfig(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null), null).get(0).getContext();
    final Boolean contextValue = (Boolean) contexts.get("IS_INCREMENTAL_HANDOFF_SUPPORTED");
    Assert.assertNotNull(contextValue);
    Assert.assertTrue(contextValue);
}

Example 2 with KafkaIndexTaskTuningConfig

use of org.apache.druid.indexing.kafka.KafkaIndexTaskTuningConfig in project druid by druid-io.

the class KafkaSupervisor method createIndexTasks.

@Override
protected List<SeekableStreamIndexTask<Integer, Long, KafkaRecordEntity>> createIndexTasks(int replicas, String baseSequenceName, ObjectMapper sortingMapper, TreeMap<Integer, Map<Integer, Long>> sequenceOffsets, SeekableStreamIndexTaskIOConfig taskIoConfig, SeekableStreamIndexTaskTuningConfig taskTuningConfig, RowIngestionMetersFactory rowIngestionMetersFactory) throws JsonProcessingException {
    final String checkpoints = sortingMapper.writerFor(CHECKPOINTS_TYPE_REF).writeValueAsString(sequenceOffsets);
    final Map<String, Object> context = createBaseTaskContexts();
    context.put(CHECKPOINTS_CTX_KEY, checkpoints);
    // Kafka index task always uses incremental handoff since 0.16.0.
    // The below is for the compatibility when you want to downgrade your cluster to something earlier than 0.16.0.
    // Kafka index task will pick up LegacyKafkaIndexTaskRunner without the below configuration.
    context.put("IS_INCREMENTAL_HANDOFF_SUPPORTED", true);
    List<SeekableStreamIndexTask<Integer, Long, KafkaRecordEntity>> taskList = new ArrayList<>();
    for (int i = 0; i < replicas; i++) {
        String taskId = IdUtils.getRandomIdWithPrefix(baseSequenceName);
        taskList.add(new KafkaIndexTask(taskId, new TaskResource(baseSequenceName, 1), spec.getDataSchema(), (KafkaIndexTaskTuningConfig) taskTuningConfig, (KafkaIndexTaskIOConfig) taskIoConfig, context, sortingMapper));
    }
    return taskList;
}