use of software.amazon.kinesis.retrieval.RetrievalConfig in project aws-glue-schema-registry by awslabs.
the class GlueSchemaRegistryKinesisIntegrationTest method startConsumingWithKCL.
private Scheduler startConsumingWithKCL(GlueSchemaRegistryConfiguration gsrConfig, RecordProcessor recordProcessor) throws InterruptedException {
GlueSchemaRegistryRecordProcessorFactory glueSchemaRegistryRecordProcessorFactory = new GlueSchemaRegistryRecordProcessorFactory(recordProcessor, glueSchemaRegistryDeserializerFactory, gsrConfig);
ConfigsBuilder configsBuilder = new ConfigsBuilder(streamName, streamName, kinesisClient, dynamoClient, cloudWatchClient, streamName, glueSchemaRegistryRecordProcessorFactory);
RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig().retrievalSpecificConfig(new PollingConfig(streamName, kinesisClient));
Scheduler scheduler = new Scheduler(configsBuilder.checkpointConfig(), configsBuilder.coordinatorConfig(), configsBuilder.leaseManagementConfig(), configsBuilder.lifecycleConfig(), configsBuilder.metricsConfig().metricsFactory(new NullMetricsFactory()), configsBuilder.processorConfig(), retrievalConfig);
new Thread(scheduler).start();
TimeUnit.SECONDS.sleep(KCL_SCHEDULER_START_UP_WAIT_TIME_SECONDS);
return scheduler;
}
use of software.amazon.kinesis.retrieval.RetrievalConfig in project pulsar by apache.
the class KinesisSource method open.
@Override
public void open(Map<String, Object> config, SourceContext sourceContext) throws Exception {
this.kinesisSourceConfig = IOConfigUtils.loadWithSecrets(config, KinesisSourceConfig.class, sourceContext);
checkArgument(isNotBlank(kinesisSourceConfig.getAwsKinesisStreamName()), "empty kinesis-stream name");
checkArgument(isNotBlank(kinesisSourceConfig.getAwsEndpoint()) || isNotBlank(kinesisSourceConfig.getAwsRegion()), "Either the aws-end-point or aws-region must be set");
checkArgument(isNotBlank(kinesisSourceConfig.getAwsCredentialPluginParam()), "empty aws-credential param");
if (kinesisSourceConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) {
checkArgument((kinesisSourceConfig.getStartAtTime() != null), "Timestamp must be specified");
}
queue = new LinkedBlockingQueue<>(kinesisSourceConfig.getReceiveQueueSize());
workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID();
AwsCredentialProviderPlugin credentialsProvider = createCredentialProvider(kinesisSourceConfig.getAwsCredentialPluginName(), kinesisSourceConfig.getAwsCredentialPluginParam());
KinesisAsyncClient kClient = kinesisSourceConfig.buildKinesisAsyncClient(credentialsProvider);
recordProcessorFactory = new KinesisRecordProcessorFactory(queue, kinesisSourceConfig);
configsBuilder = new ConfigsBuilder(kinesisSourceConfig.getAwsKinesisStreamName(), kinesisSourceConfig.getApplicationName(), kClient, kinesisSourceConfig.buildDynamoAsyncClient(credentialsProvider), kinesisSourceConfig.buildCloudwatchAsyncClient(credentialsProvider), workerId, recordProcessorFactory);
RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig();
if (!kinesisSourceConfig.isUseEnhancedFanOut()) {
retrievalConfig.retrievalSpecificConfig(new PollingConfig(kinesisSourceConfig.getAwsKinesisStreamName(), kClient));
}
retrievalConfig.initialPositionInStreamExtended(kinesisSourceConfig.getStreamStartPosition());
scheduler = new Scheduler(configsBuilder.checkpointConfig(), configsBuilder.coordinatorConfig(), configsBuilder.leaseManagementConfig(), configsBuilder.lifecycleConfig(), configsBuilder.metricsConfig(), configsBuilder.processorConfig(), retrievalConfig);
schedulerThread = new Thread(scheduler);
schedulerThread.setDaemon(true);
threadEx = null;
schedulerThread.setUncaughtExceptionHandler((t, ex) -> {
threadEx = ex;
});
schedulerThread.start();
}
Aggregations