Examples with OffsetCheckpoint org.apache.kafka.streams.state.internals.OffsetCheckpoint used on opensource projects

Example 16 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project apache-kafka-on-k8s by banzaicloud.

the class StreamTaskTest method shouldCheckpointOffsetsOnCommit.

@Test
public void shouldCheckpointOffsetsOnCommit() throws IOException {
    task = createStatefulTask(false, true);
    task.initializeStateStores();
    task.initializeTopology();
    task.commit();
    final OffsetCheckpoint checkpoint = new OffsetCheckpoint(new File(stateDirectory.directoryForTask(taskId00), ProcessorStateManager.CHECKPOINT_FILE_NAME));
    assertThat(checkpoint.read(), equalTo(Collections.singletonMap(changelogPartition, offset)));
}

Example 17 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project apache-kafka-on-k8s by banzaicloud.

the class StandbyTaskTest method testUpdate.

@Test
public void testUpdate() throws IOException {
    StreamsConfig config = createConfig(baseDir);
    StandbyTask task = new StandbyTask(taskId, topicPartitions, topology, consumer, changelogReader, config, null, stateDirectory);
    task.initializeStateStores();
    final Set<TopicPartition> partition = Collections.singleton(partition2);
    restoreStateConsumer.assign(partition);
    for (ConsumerRecord<Integer, Integer> record : Arrays.asList(new ConsumerRecord<>(partition2.topic(), partition2.partition(), 10, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, 1, 100), new ConsumerRecord<>(partition2.topic(), partition2.partition(), 20, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, 2, 100), new ConsumerRecord<>(partition2.topic(), partition2.partition(), 30, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, 3, 100))) {
        restoreStateConsumer.bufferRecord(record);
    }
    restoreStateConsumer.seekToBeginning(partition);
    task.update(partition2, restoreStateConsumer.poll(100).records(partition2));
    StandbyContextImpl context = (StandbyContextImpl) task.context();
    MockStateStore store1 = (MockStateStore) context.getStateMgr().getStore(storeName1);
    MockStateStore store2 = (MockStateStore) context.getStateMgr().getStore(storeName2);
    assertEquals(Collections.emptyList(), store1.keys);
    assertEquals(Utils.mkList(1, 2, 3), store2.keys);
    task.closeStateManager(true);
    File taskDir = stateDirectory.directoryForTask(taskId);
    OffsetCheckpoint checkpoint = new OffsetCheckpoint(new File(taskDir, ProcessorStateManager.CHECKPOINT_FILE_NAME));
    Map<TopicPartition, Long> offsets = checkpoint.read();
    assertEquals(1, offsets.size());
    assertEquals(new Long(30L + 1L), offsets.get(partition2));
}

Example 18 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project apache-kafka-on-k8s by banzaicloud.

the class StandbyTaskTest method testUpdateKTable.

@Test
public void testUpdateKTable() throws IOException {
    consumer.assign(Utils.mkList(globalTopicPartition));
    Map<TopicPartition, OffsetAndMetadata> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition(globalTopicPartition.topic(), globalTopicPartition.partition()), new OffsetAndMetadata(0L));
    consumer.commitSync(committedOffsets);
    restoreStateConsumer.updatePartitions(globalStoreName, Utils.mkList(new PartitionInfo(globalStoreName, 0, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo(globalStoreName, 1, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo(globalStoreName, 2, Node.noNode(), new Node[0], new Node[0])));
    StreamsConfig config = createConfig(baseDir);
    StandbyTask task = new StandbyTask(taskId, ktablePartitions, ktableTopology, consumer, changelogReader, config, null, stateDirectory);
    task.initializeStateStores();
    restoreStateConsumer.assign(new ArrayList<>(task.checkpointedOffsets().keySet()));
    for (ConsumerRecord<Integer, Integer> record : Arrays.asList(new ConsumerRecord<>(globalTopicPartition.topic(), globalTopicPartition.partition(), 10, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, 1, 100), new ConsumerRecord<>(globalTopicPartition.topic(), globalTopicPartition.partition(), 20, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, 2, 100), new ConsumerRecord<>(globalTopicPartition.topic(), globalTopicPartition.partition(), 30, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, 3, 100), new ConsumerRecord<>(globalTopicPartition.topic(), globalTopicPartition.partition(), 40, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, 4, 100), new ConsumerRecord<>(globalTopicPartition.topic(), globalTopicPartition.partition(), 50, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, 5, 100))) {
        restoreStateConsumer.bufferRecord(record);
    }
    for (Map.Entry<TopicPartition, Long> entry : task.checkpointedOffsets().entrySet()) {
        TopicPartition partition = entry.getKey();
        long offset = entry.getValue();
        if (offset >= 0) {
            restoreStateConsumer.seek(partition, offset);
        } else {
            restoreStateConsumer.seekToBeginning(singleton(partition));
        }
    }
    // The commit offset is at 0L. Records should not be processed
    List<ConsumerRecord<byte[], byte[]>> remaining = task.update(globalTopicPartition, restoreStateConsumer.poll(100).records(globalTopicPartition));
    assertEquals(5, remaining.size());
    committedOffsets.put(new TopicPartition(globalTopicPartition.topic(), globalTopicPartition.partition()), new OffsetAndMetadata(10L));
    consumer.commitSync(committedOffsets);
    // update offset limits
    task.commit();
    // The commit offset has not reached, yet.
    remaining = task.update(globalTopicPartition, remaining);
    assertEquals(5, remaining.size());
    committedOffsets.put(new TopicPartition(globalTopicPartition.topic(), globalTopicPartition.partition()), new OffsetAndMetadata(11L));
    consumer.commitSync(committedOffsets);
    // update offset limits
    task.commit();
    // one record should be processed.
    remaining = task.update(globalTopicPartition, remaining);
    assertEquals(4, remaining.size());
    committedOffsets.put(new TopicPartition(globalTopicPartition.topic(), globalTopicPartition.partition()), new OffsetAndMetadata(45L));
    consumer.commitSync(committedOffsets);
    // update offset limits
    task.commit();
    // The commit offset is now 45. All record except for the last one should be processed.
    remaining = task.update(globalTopicPartition, remaining);
    assertEquals(1, remaining.size());
    committedOffsets.put(new TopicPartition(globalTopicPartition.topic(), globalTopicPartition.partition()), new OffsetAndMetadata(50L));
    consumer.commitSync(committedOffsets);
    // update offset limits
    task.commit();
    // The commit offset is now 50. Still the last record remains.
    remaining = task.update(globalTopicPartition, remaining);
    assertEquals(1, remaining.size());
    committedOffsets.put(new TopicPartition(globalTopicPartition.topic(), globalTopicPartition.partition()), new OffsetAndMetadata(60L));
    consumer.commitSync(committedOffsets);
    // update offset limits
    task.commit();
    // The commit offset is now 60. No record should be left.
    remaining = task.update(globalTopicPartition, remaining);
    assertNull(remaining);
    task.closeStateManager(true);
    File taskDir = stateDirectory.directoryForTask(taskId);
    OffsetCheckpoint checkpoint = new OffsetCheckpoint(new File(taskDir, ProcessorStateManager.CHECKPOINT_FILE_NAME));
    Map<TopicPartition, Long> offsets = checkpoint.read();
    assertEquals(1, offsets.size());
    assertEquals(new Long(51L), offsets.get(globalTopicPartition));
}

Example 19 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project apache-kafka-on-k8s by banzaicloud.

the class StandbyTaskTest method shouldCheckpointStoreOffsetsOnCommit.

@Test
public void shouldCheckpointStoreOffsetsOnCommit() throws IOException {
    consumer.assign(Utils.mkList(globalTopicPartition));
    final Map<TopicPartition, OffsetAndMetadata> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition(globalTopicPartition.topic(), globalTopicPartition.partition()), new OffsetAndMetadata(100L));
    consumer.commitSync(committedOffsets);
    restoreStateConsumer.updatePartitions(globalStoreName, Utils.mkList(new PartitionInfo(globalStoreName, 0, Node.noNode(), new Node[0], new Node[0])));
    final TaskId taskId = new TaskId(0, 0);
    final MockTime time = new MockTime();
    final StreamsConfig config = createConfig(baseDir);
    final StandbyTask task = new StandbyTask(taskId, ktablePartitions, ktableTopology, consumer, changelogReader, config, null, stateDirectory);
    task.initializeStateStores();
    restoreStateConsumer.assign(new ArrayList<>(task.checkpointedOffsets().keySet()));
    final byte[] serializedValue = Serdes.Integer().serializer().serialize("", 1);
    task.update(globalTopicPartition, Collections.singletonList(new ConsumerRecord<>(globalTopicPartition.topic(), globalTopicPartition.partition(), 50L, serializedValue, serializedValue)));
    time.sleep(config.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG));
    task.commit();
    final Map<TopicPartition, Long> checkpoint = new OffsetCheckpoint(new File(stateDirectory.directoryForTask(taskId), ProcessorStateManager.CHECKPOINT_FILE_NAME)).read();
    assertThat(checkpoint, equalTo(Collections.singletonMap(globalTopicPartition, 51L)));
}

Example 20 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project apache-kafka-on-k8s by banzaicloud.

the class ProcessorStateManagerTest method testChangeLogOffsets.

@Test
public void testChangeLogOffsets() throws IOException {
    final TaskId taskId = new TaskId(0, 0);
    final long lastCheckpointedOffset = 10L;
    final String storeName1 = "store1";
    final String storeName2 = "store2";
    final String storeName3 = "store3";
    final String storeTopicName1 = ProcessorStateManager.storeChangelogTopic(applicationId, storeName1);
    final String storeTopicName2 = ProcessorStateManager.storeChangelogTopic(applicationId, storeName2);
    final String storeTopicName3 = ProcessorStateManager.storeChangelogTopic(applicationId, storeName3);
    final Map<String, String> storeToChangelogTopic = new HashMap<>();
    storeToChangelogTopic.put(storeName1, storeTopicName1);
    storeToChangelogTopic.put(storeName2, storeTopicName2);
    storeToChangelogTopic.put(storeName3, storeTopicName3);
    final OffsetCheckpoint checkpoint = new OffsetCheckpoint(new File(stateDirectory.directoryForTask(taskId), ProcessorStateManager.CHECKPOINT_FILE_NAME));
    checkpoint.write(Collections.singletonMap(new TopicPartition(storeTopicName1, 0), lastCheckpointedOffset));
    final TopicPartition partition1 = new TopicPartition(storeTopicName1, 0);
    final TopicPartition partition2 = new TopicPartition(storeTopicName2, 0);
    final TopicPartition partition3 = new TopicPartition(storeTopicName3, 1);
    final MockStateStore store1 = new MockStateStore(storeName1, true);
    final MockStateStore store2 = new MockStateStore(storeName2, true);
    final MockStateStore store3 = new MockStateStore(storeName3, true);
    // if there is a source partition, inherit the partition id
    final Set<TopicPartition> sourcePartitions = Utils.mkSet(new TopicPartition(storeTopicName3, 1));
    final ProcessorStateManager stateMgr = new ProcessorStateManager(taskId, sourcePartitions, // standby
    true, stateDirectory, storeToChangelogTopic, changelogReader, false, logContext);
    try {
        stateMgr.register(store1, store1.stateRestoreCallback);
        stateMgr.register(store2, store2.stateRestoreCallback);
        stateMgr.register(store3, store3.stateRestoreCallback);
        final Map<TopicPartition, Long> changeLogOffsets = stateMgr.checkpointed();
        assertEquals(3, changeLogOffsets.size());
        assertTrue(changeLogOffsets.containsKey(partition1));
        assertTrue(changeLogOffsets.containsKey(partition2));
        assertTrue(changeLogOffsets.containsKey(partition3));
        assertEquals(lastCheckpointedOffset, (long) changeLogOffsets.get(partition1));
        assertEquals(-1L, (long) changeLogOffsets.get(partition2));
        assertEquals(-1L, (long) changeLogOffsets.get(partition3));
    } finally {
        stateMgr.close(Collections.<TopicPartition, Long>emptyMap());
    }
}