Examples with KinesisIndexTaskTuningConfig org.apache.druid.indexing.kinesis.KinesisIndexTaskTuningConfig used on opensource projects

Example 1 with KinesisIndexTaskTuningConfig

use of org.apache.druid.indexing.kinesis.KinesisIndexTaskTuningConfig in project druid by druid-io.

the class KinesisSupervisor method setupRecordSupplier.

@Override
protected RecordSupplier<String, String, ByteEntity> setupRecordSupplier() throws RuntimeException {
    KinesisSupervisorIOConfig ioConfig = spec.getIoConfig();
    KinesisIndexTaskTuningConfig taskTuningConfig = spec.getTuningConfig();
    return new KinesisRecordSupplier(KinesisRecordSupplier.getAmazonKinesisClient(ioConfig.getEndpoint(), awsCredentialsConfig, ioConfig.getAwsAssumedRoleArn(), ioConfig.getAwsExternalId()), ioConfig.getRecordsPerFetch(), ioConfig.getFetchDelayMillis(), // skip starting background fetch, it is not used
    0, ioConfig.isDeaggregate(), taskTuningConfig.getRecordBufferSize(), taskTuningConfig.getRecordBufferOfferTimeout(), taskTuningConfig.getRecordBufferFullWait(), taskTuningConfig.getFetchSequenceNumberTimeout(), taskTuningConfig.getMaxRecordsPerPoll(), ioConfig.isUseEarliestSequenceNumber());
}

Example 2 with KinesisIndexTaskTuningConfig

use of org.apache.druid.indexing.kinesis.KinesisIndexTaskTuningConfig in project druid by druid-io.

the class KinesisSupervisor method createIndexTasks.

@Override
protected List<SeekableStreamIndexTask<String, String, ByteEntity>> createIndexTasks(int replicas, String baseSequenceName, ObjectMapper sortingMapper, TreeMap<Integer, Map<String, String>> sequenceOffsets, SeekableStreamIndexTaskIOConfig taskIoConfig, SeekableStreamIndexTaskTuningConfig taskTuningConfig, RowIngestionMetersFactory rowIngestionMetersFactory) throws JsonProcessingException {
    final String checkpoints = sortingMapper.writerFor(CHECKPOINTS_TYPE_REF).writeValueAsString(sequenceOffsets);
    final Map<String, Object> context = createBaseTaskContexts();
    context.put(CHECKPOINTS_CTX_KEY, checkpoints);
    List<SeekableStreamIndexTask<String, String, ByteEntity>> taskList = new ArrayList<>();
    for (int i = 0; i < replicas; i++) {
        String taskId = IdUtils.getRandomIdWithPrefix(baseSequenceName);
        taskList.add(new KinesisIndexTask(taskId, new TaskResource(baseSequenceName, 1), spec.getDataSchema(), (KinesisIndexTaskTuningConfig) taskTuningConfig, (KinesisIndexTaskIOConfig) taskIoConfig, context, awsCredentialsConfig));
    }
    return taskList;
}