Examples with Checkpoint org.apache.samza.checkpoint.Checkpoint used on opensource projects

Example 1 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class ITestAzureCheckpointManager method testStoringAndReadingCheckpointsSamePartition.

@Test
public void testStoringAndReadingCheckpointsSamePartition() {
    Partition partition = new Partition(0);
    TaskName taskName = new TaskName("taskName0");
    SystemStreamPartition ssp = new SystemStreamPartition("Azure", "Stream", partition);
    Map<SystemStreamPartition, String> sspMap = new HashMap<>();
    sspMap.put(ssp, "12345");
    Checkpoint cp0 = new CheckpointV1(sspMap);
    sspMap.put(ssp, "54321");
    Checkpoint cp1 = new CheckpointV1(sspMap);
    checkpointManager.register(taskName);
    checkpointManager.writeCheckpoint(taskName, cp0);
    Checkpoint readCp = checkpointManager.readLastCheckpoint(taskName);
    Assert.assertEquals(cp0, readCp);
    checkpointManager.writeCheckpoint(taskName, cp1);
    Checkpoint readCp1 = checkpointManager.readLastCheckpoint(taskName);
    Assert.assertEquals(cp1, readCp1);
}

Example 2 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class ITestAzureCheckpointManager method testStoringAndReadingCheckpointsMultiPartitions.

@Test
public void testStoringAndReadingCheckpointsMultiPartitions() {
    Partition partition = new Partition(0);
    Partition partition1 = new Partition(1);
    TaskName taskName = new TaskName("taskName");
    SystemStreamPartition ssp = new SystemStreamPartition("Azure", "Stream", partition);
    SystemStreamPartition ssp1 = new SystemStreamPartition("Azure", "Stream", partition1);
    Map<SystemStreamPartition, String> sspMap = new HashMap<>();
    sspMap.put(ssp, "12345");
    sspMap.put(ssp1, "54321");
    Checkpoint cp1 = new CheckpointV1(sspMap);
    Map<SystemStreamPartition, String> sspMap2 = new HashMap<>();
    sspMap2.put(ssp, "12347");
    sspMap2.put(ssp1, "54323");
    Checkpoint cp2 = new CheckpointV1(sspMap2);
    checkpointManager.register(taskName);
    checkpointManager.writeCheckpoint(taskName, cp1);
    Checkpoint readCp1 = checkpointManager.readLastCheckpoint(taskName);
    Assert.assertEquals(cp1, readCp1);
    checkpointManager.writeCheckpoint(taskName, cp2);
    Checkpoint readCp2 = checkpointManager.readLastCheckpoint(taskName);
    Assert.assertEquals(cp2, readCp2);
}

Example 3 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class ITestAzureCheckpointManager method testMultipleBatchWrites.

@Test
public void testMultipleBatchWrites() {
    TaskName taskName = new TaskName("taskName3");
    Map<SystemStreamPartition, String> sspMap = new HashMap<>();
    final int testBatchNum = 2;
    final int testOffsetNum = testBatchNum * AzureCheckpointManager.MAX_WRITE_BATCH_SIZE;
    for (int i = 0; i < testOffsetNum; i++) {
        Partition partition = new Partition(i);
        SystemStreamPartition ssp = new SystemStreamPartition("Azure", "Stream", partition);
        sspMap.put(ssp, String.valueOf(i));
    }
    Checkpoint cp0 = new CheckpointV1(sspMap);
    checkpointManager.register(taskName);
    checkpointManager.writeCheckpoint(taskName, cp0);
    Checkpoint readCp = checkpointManager.readLastCheckpoint(taskName);
    Assert.assertEquals(cp0, readCp);
}

Example 4 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TestKafkaCheckpointManager method testReadCheckpointV1.

@Test
public void testReadCheckpointV1() throws InterruptedException {
    setupSystemFactory(config());
    CheckpointV1 checkpointV1 = buildCheckpointV1(INPUT_SSP0, "0");
    List<IncomingMessageEnvelope> checkpointEnvelopes = ImmutableList.of(newCheckpointV1Envelope(TASK0, checkpointV1, "0"));
    setupConsumer(checkpointEnvelopes);
    KafkaCheckpointManager kafkaCheckpointManager = buildKafkaCheckpointManager(true, config());
    kafkaCheckpointManager.register(TASK0);
    Checkpoint actualCheckpoint = kafkaCheckpointManager.readLastCheckpoint(TASK0);
    assertEquals(checkpointV1, actualCheckpoint);
}

Example 5 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TestKafkaCheckpointManager method testReadIgnoreCheckpointV2WhenV1Enabled.

@Test
public void testReadIgnoreCheckpointV2WhenV1Enabled() throws InterruptedException {
    setupSystemFactory(config());
    CheckpointV1 checkpointV1 = buildCheckpointV1(INPUT_SSP0, "0");
    List<IncomingMessageEnvelope> checkpointEnvelopes = ImmutableList.of(newCheckpointV1Envelope(TASK0, checkpointV1, "0"), newCheckpointV2Envelope(TASK0, buildCheckpointV2(INPUT_SSP0, "1"), "1"));
    setupConsumer(checkpointEnvelopes);
    // default is to only read CheckpointV1
    KafkaCheckpointManager kafkaCheckpointManager = buildKafkaCheckpointManager(true, config());
    kafkaCheckpointManager.register(TASK0);
    Checkpoint actualCheckpoint = kafkaCheckpointManager.readLastCheckpoint(TASK0);
    assertEquals(checkpointV1, actualCheckpoint);
}