Examples with ShutdownException com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException used on opensource projects

Example 1 with ShutdownException

use of com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException in project samza by apache.

the class TestKinesisSystemConsumer method testProcessRecordsHelper.

/**
 * Helper to simulate and test the life-cycle of record processing from a kinesis stream with a given number of shards
 * 1. Creation of record processors.
 * 2. Initialization of record processors.
 * 3. Processing records via record processors.
 * 4. Calling checkpoint on record processors.
 * 5. Shutting down (due to re-assignment or lease expiration) record processors.
 */
private void testProcessRecordsHelper(String system, String stream, int numShards, int numRecordsPerShard) throws InterruptedException, NoSuchFieldException, IllegalAccessException {
    KinesisConfig kConfig = new KinesisConfig(new MapConfig());
    // Create consumer
    KinesisSystemConsumer consumer = new KinesisSystemConsumer(system, kConfig, new NoOpMetricsRegistry());
    initializeMetrics(consumer, stream);
    List<SystemStreamPartition> ssps = new LinkedList<>();
    IntStream.range(0, numShards).forEach(p -> {
        SystemStreamPartition ssp = new SystemStreamPartition(system, stream, new Partition(p));
        ssps.add(ssp);
    });
    ssps.forEach(ssp -> consumer.register(ssp, SYSTEM_CONSUMER_REGISTER_OFFSET));
    // Create Kinesis record processor factory
    IRecordProcessorFactory factory = consumer.createRecordProcessorFactory(stream);
    // Create and initialize Kinesis record processor
    Map<String, KinesisRecordProcessor> processorMap = createAndInitProcessors(factory, numShards);
    List<KinesisRecordProcessor> processorList = new ArrayList<>(processorMap.values());
    // Generate records to Kinesis record processor
    Map<KinesisRecordProcessor, List<Record>> inputRecordMap = generateRecords(numRecordsPerShard, processorList);
    // Verification steps
    // Read events from the BEM queue
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> messages = readEvents(new HashSet<>(ssps), consumer, numRecordsPerShard);
    if (numRecordsPerShard > 0) {
        Assert.assertEquals(messages.size(), numShards);
    } else {
        // No input records and hence no messages
        Assert.assertEquals(messages.size(), 0);
        return;
    }
    Map<SystemStreamPartition, KinesisRecordProcessor> sspToProcessorMap = getProcessorMap(consumer);
    ssps.forEach(ssp -> {
        try {
            KinesisRecordProcessor processor = sspToProcessorMap.get(ssp);
            // Verify that the read messages are received in order and are the same as input records
            Assert.assertEquals(messages.get(ssp).size(), numRecordsPerShard);
            List<IncomingMessageEnvelope> envelopes = messages.get(ssp);
            List<Record> inputRecords = inputRecordMap.get(processor);
            verifyRecords(envelopes, inputRecords, processor.getShardId());
            // Call checkpoint on consumer and verify that the checkpoint is called with the right offset
            IncomingMessageEnvelope lastEnvelope = envelopes.get(envelopes.size() - 1);
            consumer.afterCheckpoint(Collections.singletonMap(ssp, lastEnvelope.getOffset()));
            ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
            verify(getCheckpointer(processor)).checkpoint(argument.capture());
            Assert.assertEquals(inputRecords.get(inputRecords.size() - 1).getSequenceNumber(), argument.getValue());
            // Call shutdown (with ZOMBIE reason) on processor and verify if shutdown freed the ssp mapping
            shutDownProcessor(processor, ShutdownReason.ZOMBIE);
            Assert.assertFalse(sspToProcessorMap.containsValue(processor));
            Assert.assertTrue(isSspAvailable(consumer, ssp));
        } catch (NoSuchFieldException | IllegalAccessException | InvalidStateException | ShutdownException ex) {
            throw new RuntimeException(ex);
        }
    });
}

Example 2 with ShutdownException

use of com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException in project samza by apache.

the class KinesisRecordProcessor method checkpoint.

/**
 * Invoked by the Samza thread to commit checkpoint for the shard owned by the record processor instance.
 *
 * @param seqNumber sequenceNumber to checkpoint for the shard owned by this processor instance.
 */
public void checkpoint(String seqNumber) {
    ExtendedSequenceNumber seqNumberToCheckpoint = new ExtendedSequenceNumber(seqNumber);
    if (initSeqNumber.compareTo(seqNumberToCheckpoint) > 0) {
        LOG.warn("Samza called checkpoint with seqNumber {} smaller than initial seqNumber {} for {}. Ignoring it!", seqNumber, initSeqNumber, this);
        return;
    }
    if (checkpointer == null) {
        // checkpointer could be null as a result of shard re-assignment before the first record is processed.
        LOG.warn("Ignoring checkpointing for {} with seqNumber {} because of re-assignment.", this, seqNumber);
        return;
    }
    try {
        checkpointer.checkpoint(seqNumber);
        lastCheckpointedRecordSeqNumber = seqNumberToCheckpoint;
    } catch (ShutdownException e) {
        // This can happen as a result of shard re-assignment.
        String msg = String.format("Checkpointing %s with seqNumber %s failed with exception. Dropping the checkpoint.", this, seqNumber);
        LOG.warn(msg, e);
    } catch (InvalidStateException e) {
        // This can happen when KCL encounters issues with internal state, eg: dynamoDB table is not found
        String msg = String.format("Checkpointing %s with seqNumber %s failed with exception.", this, seqNumber);
        LOG.error(msg, e);
        throw new SamzaException(msg, e);
    } catch (ThrottlingException e) {
        // Throttling is handled by KCL via the client lib configuration properties. If we get an exception inspite of
        // throttling back-off behavior, let's throw an exception as the configs
        String msg = String.format("Checkpointing %s with seqNumber %s failed with exception. Checkpoint interval is" + " too aggressive for the provisioned throughput of the dynamoDB table where the checkpoints are stored." + " Either reduce the checkpoint interval -or- increase the throughput of dynamoDB table.", this, seqNumber);
        throw new SamzaException(msg);
    }
}

Example 3 with ShutdownException

use of com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException in project samza by apache.

the class TestKinesisRecordProcessor method generateRecords.

static Map<KinesisRecordProcessor, List<Record>> generateRecords(int numRecordsPerShard, List<KinesisRecordProcessor> processors) {
    Map<KinesisRecordProcessor, List<Record>> processorRecordMap = new HashMap<>();
    processors.forEach(processor -> {
        try {
            // Create records and call process records
            IRecordProcessorCheckpointer checkpointer = Mockito.mock(IRecordProcessorCheckpointer.class);
            doNothing().when(checkpointer).checkpoint(anyString());
            doNothing().when(checkpointer).checkpoint();
            ProcessRecordsInput processRecordsInput = Mockito.mock(ProcessRecordsInput.class);
            when(processRecordsInput.getCheckpointer()).thenReturn(checkpointer);
            when(processRecordsInput.getMillisBehindLatest()).thenReturn(1000L);
            List<Record> inputRecords = createRecords(numRecordsPerShard);
            processorRecordMap.put(processor, inputRecords);
            when(processRecordsInput.getRecords()).thenReturn(inputRecords);
            processor.processRecords(processRecordsInput);
        } catch (ShutdownException | InvalidStateException ex) {
            throw new RuntimeException(ex);
        }
    });
    return processorRecordMap;
}