Examples with InitializationInput com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput used on opensource projects

Example 1 with InitializationInput

use of com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput in project samza by apache.

the class TestKinesisSystemConsumer method createAndInitProcessors.

private Map<String, KinesisRecordProcessor> createAndInitProcessors(IRecordProcessorFactory factory, int numShards) {
    Map<String, KinesisRecordProcessor> processorMap = new HashMap<>();
    IntStream.range(0, numShards).forEach(p -> {
        String shardId = String.format("shard-%05d", p);
        // Create Kinesis processor
        KinesisRecordProcessor processor = (KinesisRecordProcessor) factory.createProcessor();
        // Initialize the shard
        ExtendedSequenceNumber seqNum = new ExtendedSequenceNumber("0000");
        InitializationInput initializationInput = new InitializationInput().withShardId(shardId).withExtendedSequenceNumber(seqNum);
        processor.initialize(initializationInput);
        processorMap.put(shardId, processor);
    });
    return processorMap;
}

Example 2 with InitializationInput

use of com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput in project samza by apache.

the class TestKinesisRecordProcessor method testCheckpointAfterInit.

/**
 * Test the scenario where a processor instance is created for a shard and while it is processing records, it got
 * re-assigned to the same consumer. This results in a new processor instance owning the shard and this instance
 * could receive checkpoint calls for the records that are processed by the old processor instance. This test covers
 * the scenario where the new instance receives the checkpoint call while it is done with the initialization phase and
 * before it processed any records.
 */
@Test
public void testCheckpointAfterInit() {
    String system = "kinesis";
    String stream = "stream";
    final CountDownLatch receivedShutdownLatch = new CountDownLatch(1);
    KinesisRecordProcessorListener listener = new KinesisRecordProcessorListener() {

        @Override
        public void onReceiveRecords(SystemStreamPartition ssp, List<Record> records, long millisBehindLatest) {
        }

        @Override
        public void onShutdown(SystemStreamPartition ssp) {
            receivedShutdownLatch.countDown();
        }
    };
    KinesisRecordProcessor processor = new KinesisRecordProcessor(new SystemStreamPartition(system, stream, new Partition(0)), listener);
    // Initialize the processor
    ExtendedSequenceNumber seqNum = new ExtendedSequenceNumber("0000");
    InitializationInput initializationInput = new InitializationInput().withShardId("shard-0000").withExtendedSequenceNumber(seqNum);
    processor.initialize(initializationInput);
    // Call checkpoint. This checkpoint could have originally headed to the processor instance for the same shard but
    // due to reassignment a new processor instance is created.
    processor.checkpoint("1234567");
    // Call shutdown (with ZOMBIE reason) on processor and verify that the processor calls shutdown on the listener.
    shutDownProcessor(processor, ShutdownReason.ZOMBIE);
    // Verify that the processor is shutdown.
    Assert.assertEquals("Unable to shutdown processor.", 0, receivedShutdownLatch.getCount());
}

Example 3 with InitializationInput

use of com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput in project samza by apache.

the class TestKinesisRecordProcessor method testLifeCycleHelper.

private void testLifeCycleHelper(int numRecords) {
    String system = "kinesis";
    String stream = "stream";
    final CountDownLatch receivedShutdownLatch = new CountDownLatch(1);
    final CountDownLatch receivedRecordsLatch = new CountDownLatch(numRecords > 0 ? 1 : 0);
    KinesisRecordProcessorListener listener = new KinesisRecordProcessorListener() {

        @Override
        public void onReceiveRecords(SystemStreamPartition ssp, List<Record> records, long millisBehindLatest) {
            receivedRecordsLatch.countDown();
        }

        @Override
        public void onShutdown(SystemStreamPartition ssp) {
            receivedShutdownLatch.countDown();
        }
    };
    KinesisRecordProcessor processor = new KinesisRecordProcessor(new SystemStreamPartition(system, stream, new Partition(0)), listener);
    // Initialize the processor
    ExtendedSequenceNumber seqNum = new ExtendedSequenceNumber("0000");
    InitializationInput initializationInput = new InitializationInput().withShardId("shard-0000").withExtendedSequenceNumber(seqNum);
    processor.initialize(initializationInput);
    // Call processRecords on the processor
    List<Record> records = generateRecords(numRecords, Collections.singletonList(processor)).get(processor);
    // Verification steps
    // Verify there is a receivedRecords call to listener.
    Assert.assertEquals("Unable to receive records.", 0, receivedRecordsLatch.getCount());
    if (numRecords > 0) {
        // Call checkpoint on last record
        processor.checkpoint(records.get(records.size() - 1).getSequenceNumber());
    }
    // Call shutdown (with ZOMBIE reason) on processor and verify that the processor calls shutdown on the listener.
    shutDownProcessor(processor, ShutdownReason.ZOMBIE);
    // Verify that the processor is shutdown.
    Assert.assertEquals("Unable to shutdown processor.", 0, receivedShutdownLatch.getCount());
}

Example 4 with InitializationInput

use of com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput in project samza by apache.

the class TestKinesisRecordProcessor method testShutdownDuringReshardHelper.

private void testShutdownDuringReshardHelper(int numRecords) throws InterruptedException {
    String system = "kinesis";
    String stream = "stream";
    final CountDownLatch receivedShutdownLatch = new CountDownLatch(1);
    final CountDownLatch receivedRecordsLatch = new CountDownLatch(numRecords > 0 ? 1 : 0);
    KinesisRecordProcessorListener listener = new KinesisRecordProcessorListener() {

        @Override
        public void onReceiveRecords(SystemStreamPartition ssp, List<Record> records, long millisBehindLatest) {
            receivedRecordsLatch.countDown();
        }

        @Override
        public void onShutdown(SystemStreamPartition ssp) {
            receivedShutdownLatch.countDown();
        }
    };
    KinesisRecordProcessor processor = new KinesisRecordProcessor(new SystemStreamPartition(system, stream, new Partition(0)), listener);
    // Initialize the processor
    ExtendedSequenceNumber seqNum = new ExtendedSequenceNumber("0000");
    InitializationInput initializationInput = new InitializationInput().withShardId("shard-0000").withExtendedSequenceNumber(seqNum);
    processor.initialize(initializationInput);
    // Call processRecords on the processor
    List<Record> records = generateRecords(numRecords, Collections.singletonList(processor)).get(processor);
    // Verification steps
    // Verify there is a receivedRecords call to listener.
    Assert.assertEquals("Unable to receive records.", 0, receivedRecordsLatch.getCount());
    // Call shutdown (with TERMINATE reason) on processor and verify that the processor does not call shutdown on the
    // listener until checkpoint is called for the last record consumed from shard.
    new Thread(() -> shutDownProcessor(processor, ShutdownReason.TERMINATE)).start();
    // If there are no records, the processor should shutdown immediately.
    if (numRecords == 0) {
        Assert.assertTrue("Unable to shutdown processor.", receivedShutdownLatch.await(MAX_WAIT_TIME_SHUTDOWN_RECEIVED_MS, TimeUnit.MILLISECONDS));
        return;
    }
    Assert.assertFalse("Processor shutdown too early.", receivedShutdownLatch.await(MAX_WAIT_TIME_SHUTDOWN_RECEIVED_MS, TimeUnit.MILLISECONDS));
    // Call checkpoint for the last but one record and the processor should still not call shutdown on listener.
    processor.checkpoint(records.get(records.size() - 2).getSequenceNumber());
    Assert.assertFalse("Processor shutdown too early.", receivedShutdownLatch.await(MAX_WAIT_TIME_SHUTDOWN_RECEIVED_MS, TimeUnit.MILLISECONDS));
    // Call checkpoint for the last record and the parent partition should be removed from mapper.
    processor.checkpoint(records.get(records.size() - 1).getSequenceNumber());
    Assert.assertTrue("Unable to shutdown processor.", receivedShutdownLatch.await(MAX_WAIT_TIME_SHUTDOWN_RECEIVED_MS, TimeUnit.MILLISECONDS));
}