Examples with CheckpointV1 org.apache.samza.checkpoint.CheckpointV1 used on opensource projects

Example 1 with CheckpointV1

use of org.apache.samza.checkpoint.CheckpointV1 in project samza by apache.

the class ITestAzureCheckpointManager method testStoringAndReadingCheckpointsSamePartition.

@Test
public void testStoringAndReadingCheckpointsSamePartition() {
    Partition partition = new Partition(0);
    TaskName taskName = new TaskName("taskName0");
    SystemStreamPartition ssp = new SystemStreamPartition("Azure", "Stream", partition);
    Map<SystemStreamPartition, String> sspMap = new HashMap<>();
    sspMap.put(ssp, "12345");
    Checkpoint cp0 = new CheckpointV1(sspMap);
    sspMap.put(ssp, "54321");
    Checkpoint cp1 = new CheckpointV1(sspMap);
    checkpointManager.register(taskName);
    checkpointManager.writeCheckpoint(taskName, cp0);
    Checkpoint readCp = checkpointManager.readLastCheckpoint(taskName);
    Assert.assertEquals(cp0, readCp);
    checkpointManager.writeCheckpoint(taskName, cp1);
    Checkpoint readCp1 = checkpointManager.readLastCheckpoint(taskName);
    Assert.assertEquals(cp1, readCp1);
}

Example 2 with CheckpointV1

use of org.apache.samza.checkpoint.CheckpointV1 in project samza by apache.

the class ITestAzureCheckpointManager method testStoringAndReadingCheckpointsMultiPartitions.

@Test
public void testStoringAndReadingCheckpointsMultiPartitions() {
    Partition partition = new Partition(0);
    Partition partition1 = new Partition(1);
    TaskName taskName = new TaskName("taskName");
    SystemStreamPartition ssp = new SystemStreamPartition("Azure", "Stream", partition);
    SystemStreamPartition ssp1 = new SystemStreamPartition("Azure", "Stream", partition1);
    Map<SystemStreamPartition, String> sspMap = new HashMap<>();
    sspMap.put(ssp, "12345");
    sspMap.put(ssp1, "54321");
    Checkpoint cp1 = new CheckpointV1(sspMap);
    Map<SystemStreamPartition, String> sspMap2 = new HashMap<>();
    sspMap2.put(ssp, "12347");
    sspMap2.put(ssp1, "54323");
    Checkpoint cp2 = new CheckpointV1(sspMap2);
    checkpointManager.register(taskName);
    checkpointManager.writeCheckpoint(taskName, cp1);
    Checkpoint readCp1 = checkpointManager.readLastCheckpoint(taskName);
    Assert.assertEquals(cp1, readCp1);
    checkpointManager.writeCheckpoint(taskName, cp2);
    Checkpoint readCp2 = checkpointManager.readLastCheckpoint(taskName);
    Assert.assertEquals(cp2, readCp2);
}

Example 3 with CheckpointV1

use of org.apache.samza.checkpoint.CheckpointV1 in project samza by apache.

the class ITestAzureCheckpointManager method testMultipleBatchWrites.

@Test
public void testMultipleBatchWrites() {
    TaskName taskName = new TaskName("taskName3");
    Map<SystemStreamPartition, String> sspMap = new HashMap<>();
    final int testBatchNum = 2;
    final int testOffsetNum = testBatchNum * AzureCheckpointManager.MAX_WRITE_BATCH_SIZE;
    for (int i = 0; i < testOffsetNum; i++) {
        Partition partition = new Partition(i);
        SystemStreamPartition ssp = new SystemStreamPartition("Azure", "Stream", partition);
        sspMap.put(ssp, String.valueOf(i));
    }
    Checkpoint cp0 = new CheckpointV1(sspMap);
    checkpointManager.register(taskName);
    checkpointManager.writeCheckpoint(taskName, cp0);
    Checkpoint readCp = checkpointManager.readLastCheckpoint(taskName);
    Assert.assertEquals(cp0, readCp);
}

Example 4 with CheckpointV1

use of org.apache.samza.checkpoint.CheckpointV1 in project samza by apache.

the class TestKafkaCheckpointManager method testWriteCheckpointV1.

@Test
public void testWriteCheckpointV1() {
    setupSystemFactory(config());
    KafkaCheckpointManager kafkaCheckpointManager = buildKafkaCheckpointManager(true, config());
    kafkaCheckpointManager.register(TASK0);
    CheckpointV1 checkpointV1 = buildCheckpointV1(INPUT_SSP0, "0");
    kafkaCheckpointManager.writeCheckpoint(TASK0, checkpointV1);
    ArgumentCaptor<OutgoingMessageEnvelope> outgoingMessageEnvelopeArgumentCaptor = ArgumentCaptor.forClass(OutgoingMessageEnvelope.class);
    verify(this.systemProducer).send(eq(TASK0.getTaskName()), outgoingMessageEnvelopeArgumentCaptor.capture());
    assertEquals(CHECKPOINT_SSP, outgoingMessageEnvelopeArgumentCaptor.getValue().getSystemStream());
    assertEquals(new KafkaCheckpointLogKey(KafkaCheckpointLogKey.CHECKPOINT_V1_KEY_TYPE, TASK0, GROUPER_FACTORY_CLASS), KAFKA_CHECKPOINT_LOG_KEY_SERDE.fromBytes((byte[]) outgoingMessageEnvelopeArgumentCaptor.getValue().getKey()));
    assertEquals(checkpointV1, CHECKPOINT_V1_SERDE.fromBytes((byte[]) outgoingMessageEnvelopeArgumentCaptor.getValue().getMessage()));
    verify(this.systemProducer).flush(TASK0.getTaskName());
}

Example 5 with CheckpointV1

use of org.apache.samza.checkpoint.CheckpointV1 in project samza by apache.

the class TestKafkaCheckpointManager method testConsumerStopsAfterInitialReadDisabled.

@Test
public void testConsumerStopsAfterInitialReadDisabled() throws Exception {
    Config config = config(ImmutableMap.of(TaskConfig.INTERNAL_CHECKPOINT_MANAGER_CONSUMER_STOP_AFTER_FIRST_READ, "false"));
    setupSystemFactory(config);
    // 1) return checkpointV1 for INPUT_SSP
    CheckpointV1 ssp0FirstCheckpointV1 = buildCheckpointV1(INPUT_SSP0, "0");
    List<IncomingMessageEnvelope> checkpointEnvelopes0 = ImmutableList.of(newCheckpointV1Envelope(TASK0, buildCheckpointV1(INPUT_SSP0, "0"), "0"));
    setupConsumer(checkpointEnvelopes0);
    KafkaCheckpointManager kafkaCheckpointManager = buildKafkaCheckpointManager(true, config);
    kafkaCheckpointManager.register(TASK0);
    assertEquals(ssp0FirstCheckpointV1, kafkaCheckpointManager.readLastCheckpoint(TASK0));
    // 2) return new checkpointV1 for just INPUT_SSP
    CheckpointV1 ssp0SecondCheckpointV1 = buildCheckpointV1(INPUT_SSP0, "10");
    List<IncomingMessageEnvelope> checkpointEnvelopes1 = ImmutableList.of(newCheckpointV1Envelope(TASK0, ssp0SecondCheckpointV1, "1"));
    setupConsumer(checkpointEnvelopes1);
    assertEquals(ssp0SecondCheckpointV1, kafkaCheckpointManager.readLastCheckpoint(TASK0));
    verify(this.systemConsumer, never()).stop();
}