Examples with KinesisRecordSupplier org.apache.druid.indexing.kinesis.KinesisRecordSupplier used on opensource projects

Example 1 with KinesisRecordSupplier

use of org.apache.druid.indexing.kinesis.KinesisRecordSupplier in project druid by druid-io.

the class KinesisSupervisor method setupRecordSupplier.

@Override
protected RecordSupplier<String, String, ByteEntity> setupRecordSupplier() throws RuntimeException {
    KinesisSupervisorIOConfig ioConfig = spec.getIoConfig();
    KinesisIndexTaskTuningConfig taskTuningConfig = spec.getTuningConfig();
    return new KinesisRecordSupplier(KinesisRecordSupplier.getAmazonKinesisClient(ioConfig.getEndpoint(), awsCredentialsConfig, ioConfig.getAwsAssumedRoleArn(), ioConfig.getAwsExternalId()), ioConfig.getRecordsPerFetch(), ioConfig.getFetchDelayMillis(), // skip starting background fetch, it is not used
    0, ioConfig.isDeaggregate(), taskTuningConfig.getRecordBufferSize(), taskTuningConfig.getRecordBufferOfferTimeout(), taskTuningConfig.getRecordBufferFullWait(), taskTuningConfig.getFetchSequenceNumberTimeout(), taskTuningConfig.getMaxRecordsPerPoll(), ioConfig.isUseEarliestSequenceNumber());
}

Example 2 with KinesisRecordSupplier

use of org.apache.druid.indexing.kinesis.KinesisRecordSupplier in project druid by druid-io.

the class KinesisSupervisorTest method testRecordSupplier.

@Test
public void testRecordSupplier() {
    KinesisSupervisorIOConfig kinesisSupervisorIOConfig = new KinesisSupervisorIOConfig(STREAM, INPUT_FORMAT, "awsEndpoint", null, 1, 1, new Period("PT30M"), new Period("P1D"), new Period("PT30S"), false, new Period("PT30M"), null, null, null, 100, 1000, null, null, null, false);
    KinesisIndexTaskClientFactory clientFactory = new KinesisIndexTaskClientFactory(null, OBJECT_MAPPER);
    KinesisSupervisor supervisor = new KinesisSupervisor(taskStorage, taskMaster, indexerMetadataStorageCoordinator, clientFactory, OBJECT_MAPPER, new KinesisSupervisorSpec(null, dataSchema, tuningConfig, kinesisSupervisorIOConfig, null, false, taskStorage, taskMaster, indexerMetadataStorageCoordinator, clientFactory, OBJECT_MAPPER, new NoopServiceEmitter(), new DruidMonitorSchedulerConfig(), rowIngestionMetersFactory, null, new SupervisorStateManagerConfig()), rowIngestionMetersFactory, null);
    KinesisRecordSupplier supplier = (KinesisRecordSupplier) supervisor.setupRecordSupplier();
    Assert.assertNotNull(supplier);
    Assert.assertEquals(0, supplier.bufferSize());
    Assert.assertEquals(Collections.emptySet(), supplier.getAssignment());
    // background fetch should not be enabled for supervisor supplier
    supplier.start();
    Assert.assertFalse(supplier.isBackgroundFetchRunning());
}

Example 3 with KinesisRecordSupplier

use of org.apache.druid.indexing.kinesis.KinesisRecordSupplier in project druid by druid-io.

the class KinesisSupervisor method updatePartitionLagFromStream.

@Override
protected void updatePartitionLagFromStream() {
    KinesisRecordSupplier supplier = (KinesisRecordSupplier) recordSupplier;
    // this recordSupplier method is thread safe, so does not need to acquire the recordSupplierLock
    currentPartitionTimeLag = supplier.getPartitionsTimeLag(getIoConfig().getStream(), getHighestCurrentOffsets());
}

Example 4 with KinesisRecordSupplier

use of org.apache.druid.indexing.kinesis.KinesisRecordSupplier in project druid by druid-io.

the class KinesisSupervisor method updateClosedShardCache.

private synchronized void updateClosedShardCache() {
    final KinesisRecordSupplier kinesisRecordSupplier = (KinesisRecordSupplier) recordSupplier;
    final String stream = spec.getSource();
    final Set<Shard> allActiveShards = kinesisRecordSupplier.getShards(stream);
    final Set<String> activeClosedShards = allActiveShards.stream().filter(shard -> isShardClosed(shard)).map(Shard::getShardId).collect(Collectors.toSet());
    // clear stale shards
    emptyClosedShardIds.retainAll(activeClosedShards);
    nonEmptyClosedShardIds.retainAll(activeClosedShards);
    for (String closedShardId : activeClosedShards) {
        // Try to utilize cache
        if (emptyClosedShardIds.contains(closedShardId) || nonEmptyClosedShardIds.contains(closedShardId)) {
            continue;
        }
        // Check if it is closed using kinesis and add to cache
        if (kinesisRecordSupplier.isClosedShardEmpty(stream, closedShardId)) {
            emptyClosedShardIds.add(closedShardId);
        } else {
            nonEmptyClosedShardIds.add(closedShardId);
        }
    }
}