Examples with AwsCredentialProviderPlugin org.apache.pulsar.io.aws.AwsCredentialProviderPlugin used on opensource projects

Example 1 with AwsCredentialProviderPlugin

use of org.apache.pulsar.io.aws.AwsCredentialProviderPlugin in project pulsar by apache.

the class DynamoDBSource method open.

@Override
public void open(Map<String, Object> config, SourceContext sourceContext) throws Exception {
    this.dynamodbSourceConfig = DynamoDBSourceConfig.load(config);
    checkArgument(isNotBlank(dynamodbSourceConfig.getAwsDynamodbStreamArn()), "empty dynamo-stream arn");
    // Even if the endpoint is set, it seems to require a region to go with it
    checkArgument(isNotBlank(dynamodbSourceConfig.getAwsRegion()), "The aws-region must be set");
    checkArgument(isNotBlank(dynamodbSourceConfig.getAwsCredentialPluginParam()), "empty aws-credential param");
    if (dynamodbSourceConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) {
        checkArgument((dynamodbSourceConfig.getStartAtTime() != null), "Timestamp must be specified");
    }
    queue = new LinkedBlockingQueue<>(dynamodbSourceConfig.getReceiveQueueSize());
    workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID();
    AwsCredentialProviderPlugin credentialsProvider = createCredentialProvider(dynamodbSourceConfig.getAwsCredentialPluginName(), dynamodbSourceConfig.getAwsCredentialPluginParam());
    AmazonDynamoDBStreams dynamoDBStreamsClient = dynamodbSourceConfig.buildDynamoDBStreamsClient(credentialsProvider);
    AmazonDynamoDBStreamsAdapterClient adapterClient = new AmazonDynamoDBStreamsAdapterClient(dynamoDBStreamsClient);
    recordProcessorFactory = new StreamsRecordProcessorFactory(queue, dynamodbSourceConfig);
    kinesisClientLibConfig = new KinesisClientLibConfiguration(dynamodbSourceConfig.getApplicationName(), dynamodbSourceConfig.getAwsDynamodbStreamArn(), credentialsProvider.getCredentialProvider(), workerId).withRegionName(dynamodbSourceConfig.getAwsRegion()).withInitialPositionInStream(dynamodbSourceConfig.getInitialPositionInStream());
    if (kinesisClientLibConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) {
        kinesisClientLibConfig.withTimestampAtInitialPositionInStream(dynamodbSourceConfig.getStartAtTime());
    }
    worker = StreamsWorkerFactory.createDynamoDbStreamsWorker(recordProcessorFactory, kinesisClientLibConfig, adapterClient, dynamodbSourceConfig.buildDynamoDBClient(credentialsProvider), dynamodbSourceConfig.buildCloudwatchClient(credentialsProvider));
    workerThread = new Thread(worker);
    workerThread.setDaemon(true);
    threadEx = null;
    workerThread.setUncaughtExceptionHandler((t, ex) -> {
        threadEx = ex;
        log.error("Worker died with error", ex);
    });
    workerThread.start();
}

Example 2 with AwsCredentialProviderPlugin

use of org.apache.pulsar.io.aws.AwsCredentialProviderPlugin in project pulsar by apache.

the class KinesisSource method open.

@Override
public void open(Map<String, Object> config, SourceContext sourceContext) throws Exception {
    this.kinesisSourceConfig = IOConfigUtils.loadWithSecrets(config, KinesisSourceConfig.class, sourceContext);
    checkArgument(isNotBlank(kinesisSourceConfig.getAwsKinesisStreamName()), "empty kinesis-stream name");
    checkArgument(isNotBlank(kinesisSourceConfig.getAwsEndpoint()) || isNotBlank(kinesisSourceConfig.getAwsRegion()), "Either the aws-end-point or aws-region must be set");
    checkArgument(isNotBlank(kinesisSourceConfig.getAwsCredentialPluginParam()), "empty aws-credential param");
    if (kinesisSourceConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) {
        checkArgument((kinesisSourceConfig.getStartAtTime() != null), "Timestamp must be specified");
    }
    queue = new LinkedBlockingQueue<>(kinesisSourceConfig.getReceiveQueueSize());
    workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID();
    AwsCredentialProviderPlugin credentialsProvider = createCredentialProvider(kinesisSourceConfig.getAwsCredentialPluginName(), kinesisSourceConfig.getAwsCredentialPluginParam());
    KinesisAsyncClient kClient = kinesisSourceConfig.buildKinesisAsyncClient(credentialsProvider);
    recordProcessorFactory = new KinesisRecordProcessorFactory(queue, kinesisSourceConfig);
    configsBuilder = new ConfigsBuilder(kinesisSourceConfig.getAwsKinesisStreamName(), kinesisSourceConfig.getApplicationName(), kClient, kinesisSourceConfig.buildDynamoAsyncClient(credentialsProvider), kinesisSourceConfig.buildCloudwatchAsyncClient(credentialsProvider), workerId, recordProcessorFactory);
    RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig();
    if (!kinesisSourceConfig.isUseEnhancedFanOut()) {
        retrievalConfig.retrievalSpecificConfig(new PollingConfig(kinesisSourceConfig.getAwsKinesisStreamName(), kClient));
    }
    retrievalConfig.initialPositionInStreamExtended(kinesisSourceConfig.getStreamStartPosition());
    scheduler = new Scheduler(configsBuilder.checkpointConfig(), configsBuilder.coordinatorConfig(), configsBuilder.leaseManagementConfig(), configsBuilder.lifecycleConfig(), configsBuilder.metricsConfig(), configsBuilder.processorConfig(), retrievalConfig);
    schedulerThread = new Thread(scheduler);
    schedulerThread.setDaemon(true);
    threadEx = null;
    schedulerThread.setUncaughtExceptionHandler((t, ex) -> {
        threadEx = ex;
    });
    schedulerThread.start();
}