Examples with KinesisClientLibConfiguration com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration used on opensource projects

Example 1 with KinesisClientLibConfiguration

use of com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration in project samza by apache.

the class TestKinesisConfig method testKclConfigs.

@Test
public void testKclConfigs() {
    Map<String, String> kv = new HashMap<>();
    String system = "kinesis";
    String stream = "kinesis-stream";
    String systemConfigPrefix = String.format("systems.%s.", system);
    // region config is required for setting kcl config.
    kv.put(systemConfigPrefix + "aws.region", "us-east-1");
    // Kcl Configs
    kv.put(systemConfigPrefix + "aws.kcl.TableName", "sample-table");
    kv.put(systemConfigPrefix + "aws.kcl.MaxRecords", "100");
    kv.put(systemConfigPrefix + "aws.kcl.CallProcessRecordsEvenForEmptyRecordList", "true");
    kv.put(systemConfigPrefix + "aws.kcl.InitialPositionInStream", "TRIM_HORIZON");
    // override one of the Kcl configs for kinesis-stream1
    kv.put(systemConfigPrefix + "streams.kinesis-stream1.aws.kcl.InitialPositionInStream", "LATEST");
    Config config = new MapConfig(kv);
    KinesisConfig kConfig = new KinesisConfig(config);
    KinesisClientLibConfiguration kclConfig = kConfig.getKinesisClientLibConfig(system, stream, "sample-app");
    assertEquals("sample-table", kclConfig.getTableName());
    assertEquals(100, kclConfig.getMaxRecords());
    assertTrue(kclConfig.shouldCallProcessRecordsEvenForEmptyRecordList());
    assertEquals(InitialPositionInStream.TRIM_HORIZON, kclConfig.getInitialPositionInStream());
    // verify if the overriden config is applied for kinesis-stream1
    kclConfig = kConfig.getKinesisClientLibConfig(system, "kinesis-stream1", "sample-app");
    assertEquals(InitialPositionInStream.LATEST, kclConfig.getInitialPositionInStream());
}

Example 2 with KinesisClientLibConfiguration

use of com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration in project aws-doc-sdk-examples by awsdocs.

the class StreamsAdapterDemo method main.

/**
 * @param args
 */
public static void main(String[] args) throws Exception {
    System.out.println("Starting demo...");
    dynamoDBClient = AmazonDynamoDBClientBuilder.standard().withRegion(awsRegion).build();
    cloudWatchClient = AmazonCloudWatchClientBuilder.standard().withRegion(awsRegion).build();
    dynamoDBStreamsClient = AmazonDynamoDBStreamsClientBuilder.standard().withRegion(awsRegion).build();
    adapterClient = new AmazonDynamoDBStreamsAdapterClient(dynamoDBStreamsClient);
    String srcTable = tablePrefix + "-src";
    String destTable = tablePrefix + "-dest";
    recordProcessorFactory = new StreamsRecordProcessorFactory(dynamoDBClient, destTable);
    setUpTables();
    workerConfig = new KinesisClientLibConfiguration("streams-adapter-demo", streamArn, awsCredentialsProvider, "streams-demo-worker").withMaxRecords(1000).withIdleTimeBetweenReadsInMillis(500).withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON);
    System.out.println("Creating worker for stream: " + streamArn);
    worker = StreamsWorkerFactory.createDynamoDbStreamsWorker(recordProcessorFactory, workerConfig, adapterClient, dynamoDBClient, cloudWatchClient);
    System.out.println("Starting worker...");
    Thread t = new Thread(worker);
    t.start();
    Thread.sleep(25000);
    worker.shutdown();
    t.join();
    if (StreamsAdapterDemoHelper.scanTable(dynamoDBClient, srcTable).getItems().equals(StreamsAdapterDemoHelper.scanTable(dynamoDBClient, destTable).getItems())) {
        System.out.println("Scan result is equal.");
    } else {
        System.out.println("Tables are different!");
    }
    System.out.println("Done.");
    cleanupAndExit(0);
}

Example 3 with KinesisClientLibConfiguration

use of com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration in project samza by apache.

the class KinesisConfig method getKinesisClientLibConfig.

/**
 * Get KCL config for a given system stream.
 * @param system name of the system
 * @param stream name of the stream
 * @param appName name of the application
 * @return Stream scoped KCL configs required to build
 *         {@link KinesisClientLibConfiguration}
 */
public KinesisClientLibConfiguration getKinesisClientLibConfig(String system, String stream, String appName) {
    ClientConfiguration clientConfig = getAWSClientConfig(system);
    String workerId = appName + "-" + UUID.randomUUID();
    InitialPositionInStream startPos = InitialPositionInStream.LATEST;
    AWSCredentialsProvider provider = credentialsProviderForStream(system, stream);
    KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(appName, stream, provider, workerId).withRegionName(getRegion(system, stream).getName()).withKinesisClientConfig(clientConfig).withCloudWatchClientConfig(clientConfig).withDynamoDBClientConfig(clientConfig).withInitialPositionInStream(startPos).withCallProcessRecordsEvenForEmptyRecordList(// For health monitoring metrics.
    true);
    // First, get system scoped configs for KCL and override with configs set at stream scope.
    setKinesisClientLibConfigs(subset(String.format(CONFIG_SYSTEM_KINESIS_CLIENT_LIB_CONFIG, system)), kinesisClientLibConfiguration);
    setKinesisClientLibConfigs(subset(String.format(CONFIG_STREAM_KINESIS_CLIENT_LIB_CONFIG, system, stream)), kinesisClientLibConfiguration);
    return kinesisClientLibConfiguration;
}