Examples with PollingConfig software.amazon.kinesis.retrieval.polling.PollingConfig used on opensource projects

Example 1 with PollingConfig

use of software.amazon.kinesis.retrieval.polling.PollingConfig in project graylog-plugin-integrations by Graylog2.

the class KinesisConsumer method run.

public void run() {
    LOG.debug("Starting the Kinesis Consumer.");
    AwsCredentialsProvider credentialsProvider = AWSAuthFactory.create(request.region(), request.awsAccessKeyId(), request.awsSecretAccessKey(), request.assumeRoleArn());
    final Region region = Region.of(request.region());
    final DynamoDbAsyncClientBuilder dynamoDbClientBuilder = DynamoDbAsyncClient.builder();
    AWSClientBuilderUtil.initializeBuilder(dynamoDbClientBuilder, request.dynamodbEndpoint(), region, credentialsProvider);
    final DynamoDbAsyncClient dynamoClient = dynamoDbClientBuilder.build();
    final CloudWatchAsyncClientBuilder cloudwatchClientBuilder = CloudWatchAsyncClient.builder();
    AWSClientBuilderUtil.initializeBuilder(cloudwatchClientBuilder, request.cloudwatchEndpoint(), region, credentialsProvider);
    final CloudWatchAsyncClient cloudWatchClient = cloudwatchClientBuilder.build();
    final KinesisAsyncClientBuilder kinesisAsyncClientBuilder = KinesisAsyncClient.builder();
    AWSClientBuilderUtil.initializeBuilder(kinesisAsyncClientBuilder, request.kinesisEndpoint(), region, credentialsProvider);
    final KinesisAsyncClient kinesisAsyncClient = KinesisClientUtil.createKinesisAsyncClient(kinesisAsyncClientBuilder);
    final String workerId = String.format(Locale.ENGLISH, "graylog-node-%s", nodeId.anonymize());
    LOG.debug("Using workerId [{}].", workerId);
    // The application name needs to be unique per input/consumer.
    final String applicationName = String.format(Locale.ENGLISH, "graylog-aws-plugin-%s", kinesisStreamName);
    LOG.debug("Using Kinesis applicationName [{}].", applicationName);
    // The KinesisShardProcessorFactory contains the message processing logic.
    final KinesisShardProcessorFactory kinesisShardProcessorFactory = new KinesisShardProcessorFactory(objectMapper, transport, handleMessageCallback, kinesisStreamName, awsMessageType);
    ConfigsBuilder configsBuilder = new ConfigsBuilder(kinesisStreamName, applicationName, kinesisAsyncClient, dynamoClient, cloudWatchClient, workerId, kinesisShardProcessorFactory);
    final PollingConfig pollingConfig = new PollingConfig(kinesisStreamName, kinesisAsyncClient);
    // Default max records per request is 10k.
    if (recordBatchSize != null) {
        LOG.debug("Using explicit batch size [{}]", recordBatchSize);
        pollingConfig.maxRecords(recordBatchSize);
    }
    this.kinesisScheduler = new Scheduler(configsBuilder.checkpointConfig(), configsBuilder.coordinatorConfig(), configsBuilder.leaseManagementConfig(), configsBuilder.lifecycleConfig(), configsBuilder.metricsConfig(), configsBuilder.processorConfig(), configsBuilder.retrievalConfig().retrievalSpecificConfig(pollingConfig));
    LOG.debug("Starting Kinesis scheduler.");
    kinesisScheduler.run();
    LOG.debug("After Kinesis scheduler stopped.");
}

Example 2 with PollingConfig

use of software.amazon.kinesis.retrieval.polling.PollingConfig in project aws-glue-schema-registry by awslabs.

the class GlueSchemaRegistryKinesisIntegrationTest method startConsumingWithKCL.

private Scheduler startConsumingWithKCL(GlueSchemaRegistryConfiguration gsrConfig, RecordProcessor recordProcessor) throws InterruptedException {
    GlueSchemaRegistryRecordProcessorFactory glueSchemaRegistryRecordProcessorFactory = new GlueSchemaRegistryRecordProcessorFactory(recordProcessor, glueSchemaRegistryDeserializerFactory, gsrConfig);
    ConfigsBuilder configsBuilder = new ConfigsBuilder(streamName, streamName, kinesisClient, dynamoClient, cloudWatchClient, streamName, glueSchemaRegistryRecordProcessorFactory);
    RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig().retrievalSpecificConfig(new PollingConfig(streamName, kinesisClient));
    Scheduler scheduler = new Scheduler(configsBuilder.checkpointConfig(), configsBuilder.coordinatorConfig(), configsBuilder.leaseManagementConfig(), configsBuilder.lifecycleConfig(), configsBuilder.metricsConfig().metricsFactory(new NullMetricsFactory()), configsBuilder.processorConfig(), retrievalConfig);
    new Thread(scheduler).start();
    TimeUnit.SECONDS.sleep(KCL_SCHEDULER_START_UP_WAIT_TIME_SECONDS);
    return scheduler;
}

Example 3 with PollingConfig

use of software.amazon.kinesis.retrieval.polling.PollingConfig in project pulsar by apache.

the class KinesisSource method open.

@Override
public void open(Map<String, Object> config, SourceContext sourceContext) throws Exception {
    this.kinesisSourceConfig = IOConfigUtils.loadWithSecrets(config, KinesisSourceConfig.class, sourceContext);
    checkArgument(isNotBlank(kinesisSourceConfig.getAwsKinesisStreamName()), "empty kinesis-stream name");
    checkArgument(isNotBlank(kinesisSourceConfig.getAwsEndpoint()) || isNotBlank(kinesisSourceConfig.getAwsRegion()), "Either the aws-end-point or aws-region must be set");
    checkArgument(isNotBlank(kinesisSourceConfig.getAwsCredentialPluginParam()), "empty aws-credential param");
    if (kinesisSourceConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) {
        checkArgument((kinesisSourceConfig.getStartAtTime() != null), "Timestamp must be specified");
    }
    queue = new LinkedBlockingQueue<>(kinesisSourceConfig.getReceiveQueueSize());
    workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID();
    AwsCredentialProviderPlugin credentialsProvider = createCredentialProvider(kinesisSourceConfig.getAwsCredentialPluginName(), kinesisSourceConfig.getAwsCredentialPluginParam());
    KinesisAsyncClient kClient = kinesisSourceConfig.buildKinesisAsyncClient(credentialsProvider);
    recordProcessorFactory = new KinesisRecordProcessorFactory(queue, kinesisSourceConfig);
    configsBuilder = new ConfigsBuilder(kinesisSourceConfig.getAwsKinesisStreamName(), kinesisSourceConfig.getApplicationName(), kClient, kinesisSourceConfig.buildDynamoAsyncClient(credentialsProvider), kinesisSourceConfig.buildCloudwatchAsyncClient(credentialsProvider), workerId, recordProcessorFactory);
    RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig();
    if (!kinesisSourceConfig.isUseEnhancedFanOut()) {
        retrievalConfig.retrievalSpecificConfig(new PollingConfig(kinesisSourceConfig.getAwsKinesisStreamName(), kClient));
    }
    retrievalConfig.initialPositionInStreamExtended(kinesisSourceConfig.getStreamStartPosition());
    scheduler = new Scheduler(configsBuilder.checkpointConfig(), configsBuilder.coordinatorConfig(), configsBuilder.leaseManagementConfig(), configsBuilder.lifecycleConfig(), configsBuilder.metricsConfig(), configsBuilder.processorConfig(), retrievalConfig);
    schedulerThread = new Thread(scheduler);
    schedulerThread.setDaemon(true);
    threadEx = null;
    schedulerThread.setUncaughtExceptionHandler((t, ex) -> {
        threadEx = ex;
    });
    schedulerThread.start();
}