Examples with KinesisProducerConfiguration com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration used on opensource projects

Example 1 with KinesisProducerConfiguration

use of com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration in project flink by apache.

the class KinesisConfigUtilTest method testThreadPoolSizeInProducerConfiguration.

@Test
public void testThreadPoolSizeInProducerConfiguration() {
    Properties testConfig = new Properties();
    testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
    KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);
    assertEquals(10, kpc.getThreadPoolSize());
    testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12");
    kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);
    assertEquals(12, kpc.getThreadPoolSize());
}

Example 2 with KinesisProducerConfiguration

use of com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration in project flink by apache.

the class KinesisConfigUtilTest method testCorrectlySetRegionInProducerConfiguration.

@Test
public void testCorrectlySetRegionInProducerConfiguration() {
    String region = "us-east-1";
    Properties testConfig = new Properties();
    testConfig.setProperty(AWSConfigConstants.AWS_REGION, region);
    KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);
    assertEquals("incorrect region", region, kpc.getRegion());
}

Example 3 with KinesisProducerConfiguration

use of com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration in project flink by apache.

the class KinesisConfigUtilTest method testRateLimitInProducerConfiguration.

@Test
public void testRateLimitInProducerConfiguration() {
    Properties testConfig = new Properties();
    testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
    KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);
    assertEquals(100, kpc.getRateLimit());
    testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150");
    kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);
    assertEquals(150, kpc.getRateLimit());
}

Example 4 with KinesisProducerConfiguration

use of com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration in project flink by apache.

the class KinesisConfigUtilTest method testThreadingModelInProducerConfiguration.

@Test
public void testThreadingModelInProducerConfiguration() {
    Properties testConfig = new Properties();
    testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
    KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);
    assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel());
    testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST");
    kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);
    assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel());
}

Example 5 with KinesisProducerConfiguration

use of com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration in project flink by apache.

the class FlinkKinesisProducer method open.

// --------------------------- Lifecycle methods ---------------------------
@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    schema.open(RuntimeContextInitializationContextAdapters.serializationAdapter(getRuntimeContext(), metricGroup -> metricGroup.addGroup("user")));
    // check and pass the configuration properties
    KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps);
    producer = getKinesisProducer(producerConfig);
    final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP);
    this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES);
    kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount);
    backpressureLatch = new TimeoutLatch();
    callback = new FutureCallback<UserRecordResult>() {

        @Override
        public void onSuccess(UserRecordResult result) {
            backpressureLatch.trigger();
            if (!result.isSuccessful()) {
                if (failOnError) {
                    // only remember the first thrown exception
                    if (thrownException == null) {
                        thrownException = new RuntimeException("Record was not sent successful");
                    }
                } else {
                    LOG.warn("Record was not sent successful");
                }
            }
        }

        @Override
        public void onFailure(Throwable t) {
            backpressureLatch.trigger();
            if (failOnError) {
                thrownException = t;
            } else {
                LOG.warn("An exception occurred while processing a record", t);
            }
        }
    };
    if (this.customPartitioner != null) {
        this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks());
    }
    final RuntimeContext ctx = getRuntimeContext();
    ctx.registerUserCodeClassLoaderReleaseHookIfAbsent(KINESIS_PRODUCER_RELEASE_HOOK_NAME, () -> this.runClassLoaderReleaseHook(ctx.getUserCodeClassLoader()));
    LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}