Examples with CheckpointV2 org.apache.samza.checkpoint.CheckpointV2 used on opensource projects

Example 21 with CheckpointV2

use of org.apache.samza.checkpoint.CheckpointV2 in project samza by apache.

the class TestKafkaCheckpointManager method testWriteCheckpointV2.

@Test
public void testWriteCheckpointV2() {
    setupSystemFactory(config());
    KafkaCheckpointManager kafkaCheckpointManager = buildKafkaCheckpointManager(true, config());
    kafkaCheckpointManager.register(TASK0);
    CheckpointV2 checkpointV2 = buildCheckpointV2(INPUT_SSP0, "0");
    kafkaCheckpointManager.writeCheckpoint(TASK0, checkpointV2);
    ArgumentCaptor<OutgoingMessageEnvelope> outgoingMessageEnvelopeArgumentCaptor = ArgumentCaptor.forClass(OutgoingMessageEnvelope.class);
    verify(this.systemProducer).send(eq(TASK0.getTaskName()), outgoingMessageEnvelopeArgumentCaptor.capture());
    assertEquals(CHECKPOINT_SSP, outgoingMessageEnvelopeArgumentCaptor.getValue().getSystemStream());
    assertEquals(new KafkaCheckpointLogKey(KafkaCheckpointLogKey.CHECKPOINT_V2_KEY_TYPE, TASK0, GROUPER_FACTORY_CLASS), KAFKA_CHECKPOINT_LOG_KEY_SERDE.fromBytes((byte[]) outgoingMessageEnvelopeArgumentCaptor.getValue().getKey()));
    assertEquals(checkpointV2, CHECKPOINT_V2_SERDE.fromBytes((byte[]) outgoingMessageEnvelopeArgumentCaptor.getValue().getMessage()));
    verify(this.systemProducer).flush(TASK0.getTaskName());
}

Example 22 with CheckpointV2

use of org.apache.samza.checkpoint.CheckpointV2 in project samza by apache.

the class TestKafkaCheckpointManager method testReadCheckpointPriority.

@Test
public void testReadCheckpointPriority() throws InterruptedException {
    Config config = config(ImmutableMap.of(TaskConfig.CHECKPOINT_READ_VERSIONS, "2,1"));
    setupSystemFactory(config);
    CheckpointV2 checkpointV2 = buildCheckpointV2(INPUT_SSP0, "1");
    List<IncomingMessageEnvelope> checkpointEnvelopes = ImmutableList.of(newCheckpointV1Envelope(TASK0, buildCheckpointV1(INPUT_SSP0, "0"), "0"), newCheckpointV2Envelope(TASK0, checkpointV2, "1"));
    setupConsumer(checkpointEnvelopes);
    KafkaCheckpointManager kafkaCheckpointManager = buildKafkaCheckpointManager(true, config);
    kafkaCheckpointManager.register(TASK0);
    Checkpoint actualCheckpoint = kafkaCheckpointManager.readLastCheckpoint(TASK0);
    assertEquals(checkpointV2, actualCheckpoint);
}

Example 23 with CheckpointV2

use of org.apache.samza.checkpoint.CheckpointV2 in project samza by apache.

the class TestKafkaCheckpointManager method testWriteCheckpointShouldRetryFiniteTimesOnFailure.

@Test
public void testWriteCheckpointShouldRetryFiniteTimesOnFailure() {
    setupSystemFactory(config());
    doThrow(new RuntimeException("send failed")).when(this.systemProducer).send(any(), any());
    KafkaCheckpointManager kafkaCheckpointManager = buildKafkaCheckpointManager(true, config());
    kafkaCheckpointManager.register(TASK0);
    // setter for scala var MaxRetryDurationInMillis
    kafkaCheckpointManager.MaxRetryDurationInMillis_$eq(100);
    CheckpointV2 checkpointV2 = buildCheckpointV2(INPUT_SSP0, "0");
    try {
        kafkaCheckpointManager.writeCheckpoint(TASK0, checkpointV2);
        fail("Expected to throw SamzaException");
    } catch (SamzaException e) {
    // expected to get here
    }
    // one call to send which fails, then writeCheckpoint gives up
    verify(this.systemProducer).send(any(), any());
    verify(this.systemProducer, never()).flush(any());
}

Example 24 with CheckpointV2

use of org.apache.samza.checkpoint.CheckpointV2 in project samza by apache.

the class TestKafkaCheckpointManager method testReadMultipleCheckpointsUpgradeCheckpointVersion.

@Test
public void testReadMultipleCheckpointsUpgradeCheckpointVersion() throws InterruptedException {
    Config config = config(ImmutableMap.of(TaskConfig.CHECKPOINT_READ_VERSIONS, "2,1"));
    setupSystemFactory(config);
    KafkaCheckpointManager kafkaCheckpointManager = buildKafkaCheckpointManager(true, config);
    kafkaCheckpointManager.register(TASK0);
    kafkaCheckpointManager.register(TASK1);
    List<IncomingMessageEnvelope> checkpointEnvelopesV1 = ImmutableList.of(newCheckpointV1Envelope(TASK0, buildCheckpointV1(INPUT_SSP0, "0"), "0"), newCheckpointV1Envelope(TASK1, buildCheckpointV1(INPUT_SSP1, "0"), "1"));
    CheckpointV2 ssp0CheckpointV2 = buildCheckpointV2(INPUT_SSP0, "10");
    CheckpointV2 ssp1CheckpointV2 = buildCheckpointV2(INPUT_SSP1, "11");
    List<IncomingMessageEnvelope> checkpointEnvelopesV2 = ImmutableList.of(newCheckpointV2Envelope(TASK0, ssp0CheckpointV2, "2"), newCheckpointV2Envelope(TASK1, ssp1CheckpointV2, "3"));
    setupConsumerMultiplePoll(ImmutableList.of(checkpointEnvelopesV1, checkpointEnvelopesV2));
    assertEquals(ssp0CheckpointV2, kafkaCheckpointManager.readLastCheckpoint(TASK0));
    assertEquals(ssp1CheckpointV2, kafkaCheckpointManager.readLastCheckpoint(TASK1));
    // 2 polls for actual checkpoints, 1 final empty poll
    verify(this.systemConsumer, times(3)).poll(ImmutableSet.of(CHECKPOINT_SSP), SystemConsumer.BLOCK_ON_OUTSTANDING_MESSAGES);
}