Examples with RetrievalConfig software.amazon.kinesis.retrieval.RetrievalConfig used on opensource projects

Example 1 with RetrievalConfig

use of software.amazon.kinesis.retrieval.RetrievalConfig in project aws-glue-schema-registry by awslabs.

the class GlueSchemaRegistryKinesisIntegrationTest method startConsumingWithKCL.

private Scheduler startConsumingWithKCL(GlueSchemaRegistryConfiguration gsrConfig, RecordProcessor recordProcessor) throws InterruptedException {
    GlueSchemaRegistryRecordProcessorFactory glueSchemaRegistryRecordProcessorFactory = new GlueSchemaRegistryRecordProcessorFactory(recordProcessor, glueSchemaRegistryDeserializerFactory, gsrConfig);
    ConfigsBuilder configsBuilder = new ConfigsBuilder(streamName, streamName, kinesisClient, dynamoClient, cloudWatchClient, streamName, glueSchemaRegistryRecordProcessorFactory);
    RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig().retrievalSpecificConfig(new PollingConfig(streamName, kinesisClient));
    Scheduler scheduler = new Scheduler(configsBuilder.checkpointConfig(), configsBuilder.coordinatorConfig(), configsBuilder.leaseManagementConfig(), configsBuilder.lifecycleConfig(), configsBuilder.metricsConfig().metricsFactory(new NullMetricsFactory()), configsBuilder.processorConfig(), retrievalConfig);
    new Thread(scheduler).start();
    TimeUnit.SECONDS.sleep(KCL_SCHEDULER_START_UP_WAIT_TIME_SECONDS);
    return scheduler;
}

Example 2 with RetrievalConfig

use of software.amazon.kinesis.retrieval.RetrievalConfig in project pulsar by apache.

the class KinesisSource method open.

@Override
public void open(Map<String, Object> config, SourceContext sourceContext) throws Exception {
    this.kinesisSourceConfig = IOConfigUtils.loadWithSecrets(config, KinesisSourceConfig.class, sourceContext);
    checkArgument(isNotBlank(kinesisSourceConfig.getAwsKinesisStreamName()), "empty kinesis-stream name");
    checkArgument(isNotBlank(kinesisSourceConfig.getAwsEndpoint()) || isNotBlank(kinesisSourceConfig.getAwsRegion()), "Either the aws-end-point or aws-region must be set");
    checkArgument(isNotBlank(kinesisSourceConfig.getAwsCredentialPluginParam()), "empty aws-credential param");
    if (kinesisSourceConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) {
        checkArgument((kinesisSourceConfig.getStartAtTime() != null), "Timestamp must be specified");
    }
    queue = new LinkedBlockingQueue<>(kinesisSourceConfig.getReceiveQueueSize());
    workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID();
    AwsCredentialProviderPlugin credentialsProvider = createCredentialProvider(kinesisSourceConfig.getAwsCredentialPluginName(), kinesisSourceConfig.getAwsCredentialPluginParam());
    KinesisAsyncClient kClient = kinesisSourceConfig.buildKinesisAsyncClient(credentialsProvider);
    recordProcessorFactory = new KinesisRecordProcessorFactory(queue, kinesisSourceConfig);
    configsBuilder = new ConfigsBuilder(kinesisSourceConfig.getAwsKinesisStreamName(), kinesisSourceConfig.getApplicationName(), kClient, kinesisSourceConfig.buildDynamoAsyncClient(credentialsProvider), kinesisSourceConfig.buildCloudwatchAsyncClient(credentialsProvider), workerId, recordProcessorFactory);
    RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig();
    if (!kinesisSourceConfig.isUseEnhancedFanOut()) {
        retrievalConfig.retrievalSpecificConfig(new PollingConfig(kinesisSourceConfig.getAwsKinesisStreamName(), kClient));
    }
    retrievalConfig.initialPositionInStreamExtended(kinesisSourceConfig.getStreamStartPosition());
    scheduler = new Scheduler(configsBuilder.checkpointConfig(), configsBuilder.coordinatorConfig(), configsBuilder.leaseManagementConfig(), configsBuilder.lifecycleConfig(), configsBuilder.metricsConfig(), configsBuilder.processorConfig(), retrievalConfig);
    schedulerThread = new Thread(scheduler);
    schedulerThread.setDaemon(true);
    threadEx = null;
    schedulerThread.setUncaughtExceptionHandler((t, ex) -> {
        threadEx = ex;
    });
    schedulerThread.start();
}