Examples with OffsetCheckpoint org.apache.kafka.streams.state.internals.OffsetCheckpoint used on opensource projects

Example 11 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project kafka by apache.

the class RestoreIntegrationTest method shouldRestoreStateFromChangelogTopic.

@Test
public void shouldRestoreStateFromChangelogTopic() throws Exception {
    final String changelog = appId + "-store-changelog";
    CLUSTER.createTopic(changelog, 2, 1);
    final AtomicInteger numReceived = new AtomicInteger(0);
    final StreamsBuilder builder = new StreamsBuilder();
    final Properties props = props();
    // restoring from 1000 to 5000, and then process from 5000 to 10000 on each of the two partitions
    final int offsetCheckpointed = 1000;
    createStateForRestoration(changelog, 0);
    createStateForRestoration(inputStream, 10000);
    final StateDirectory stateDirectory = new StateDirectory(new StreamsConfig(props), new MockTime(), true, false);
    // note here the checkpointed offset is the last processed record's offset, so without control message we should write this offset - 1
    new OffsetCheckpoint(new File(stateDirectory.getOrCreateDirectoryForTask(new TaskId(0, 0)), ".checkpoint")).write(Collections.singletonMap(new TopicPartition(changelog, 0), (long) offsetCheckpointed - 1));
    new OffsetCheckpoint(new File(stateDirectory.getOrCreateDirectoryForTask(new TaskId(0, 1)), ".checkpoint")).write(Collections.singletonMap(new TopicPartition(changelog, 1), (long) offsetCheckpointed - 1));
    final CountDownLatch startupLatch = new CountDownLatch(1);
    final CountDownLatch shutdownLatch = new CountDownLatch(1);
    builder.table(inputStream, Consumed.with(Serdes.Integer(), Serdes.Integer()), Materialized.as("store")).toStream().foreach((key, value) -> {
        if (numReceived.incrementAndGet() == numberOfKeys) {
            shutdownLatch.countDown();
        }
    });
    kafkaStreams = new KafkaStreams(builder.build(), props);
    kafkaStreams.setStateListener((newState, oldState) -> {
        if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
            startupLatch.countDown();
        }
    });
    final AtomicLong restored = new AtomicLong(0);
    kafkaStreams.setGlobalStateRestoreListener(new StateRestoreListener() {

        @Override
        public void onRestoreStart(final TopicPartition topicPartition, final String storeName, final long startingOffset, final long endingOffset) {
        }

        @Override
        public void onBatchRestored(final TopicPartition topicPartition, final String storeName, final long batchEndOffset, final long numRestored) {
        }

        @Override
        public void onRestoreEnd(final TopicPartition topicPartition, final String storeName, final long totalRestored) {
            restored.addAndGet(totalRestored);
        }
    });
    kafkaStreams.start();
    assertTrue(startupLatch.await(30, TimeUnit.SECONDS));
    assertThat(restored.get(), equalTo((long) numberOfKeys - 2 * offsetCheckpointed));
    assertTrue(shutdownLatch.await(30, TimeUnit.SECONDS));
    assertThat(numReceived.get(), equalTo(numberOfKeys));
}

Example 12 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project kafka by apache.

the class RestoreIntegrationTest method shouldRestoreStateFromSourceTopic.

@Test
public void shouldRestoreStateFromSourceTopic() throws Exception {
    final AtomicInteger numReceived = new AtomicInteger(0);
    final StreamsBuilder builder = new StreamsBuilder();
    final Properties props = props();
    props.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.OPTIMIZE);
    // restoring from 1000 to 4000 (committed), and then process from 4000 to 5000 on each of the two partitions
    final int offsetLimitDelta = 1000;
    final int offsetCheckpointed = 1000;
    createStateForRestoration(inputStream, 0);
    setCommittedOffset(inputStream, offsetLimitDelta);
    final StateDirectory stateDirectory = new StateDirectory(new StreamsConfig(props), new MockTime(), true, false);
    // note here the checkpointed offset is the last processed record's offset, so without control message we should write this offset - 1
    new OffsetCheckpoint(new File(stateDirectory.getOrCreateDirectoryForTask(new TaskId(0, 0)), ".checkpoint")).write(Collections.singletonMap(new TopicPartition(inputStream, 0), (long) offsetCheckpointed - 1));
    new OffsetCheckpoint(new File(stateDirectory.getOrCreateDirectoryForTask(new TaskId(0, 1)), ".checkpoint")).write(Collections.singletonMap(new TopicPartition(inputStream, 1), (long) offsetCheckpointed - 1));
    final CountDownLatch startupLatch = new CountDownLatch(1);
    final CountDownLatch shutdownLatch = new CountDownLatch(1);
    builder.table(inputStream, Materialized.<Integer, Integer, KeyValueStore<Bytes, byte[]>>as("store").withKeySerde(Serdes.Integer()).withValueSerde(Serdes.Integer())).toStream().foreach((key, value) -> {
        if (numReceived.incrementAndGet() == offsetLimitDelta * 2) {
            shutdownLatch.countDown();
        }
    });
    kafkaStreams = new KafkaStreams(builder.build(props), props);
    kafkaStreams.setStateListener((newState, oldState) -> {
        if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
            startupLatch.countDown();
        }
    });
    final AtomicLong restored = new AtomicLong(0);
    kafkaStreams.setGlobalStateRestoreListener(new StateRestoreListener() {

        @Override
        public void onRestoreStart(final TopicPartition topicPartition, final String storeName, final long startingOffset, final long endingOffset) {
        }

        @Override
        public void onBatchRestored(final TopicPartition topicPartition, final String storeName, final long batchEndOffset, final long numRestored) {
        }

        @Override
        public void onRestoreEnd(final TopicPartition topicPartition, final String storeName, final long totalRestored) {
            restored.addAndGet(totalRestored);
        }
    });
    kafkaStreams.start();
    assertTrue(startupLatch.await(30, TimeUnit.SECONDS));
    assertThat(restored.get(), equalTo((long) numberOfKeys - offsetLimitDelta * 2 - offsetCheckpointed * 2));
    assertTrue(shutdownLatch.await(30, TimeUnit.SECONDS));
    assertThat(numReceived.get(), equalTo(offsetLimitDelta * 2));
}

Example 13 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project kafka by apache.

the class ProcessorStateManagerTest method testChangeLogOffsets.

@Test
public void testChangeLogOffsets() throws IOException {
    final TaskId taskId = new TaskId(0, 0);
    long lastCheckpointedOffset = 10L;
    String storeName1 = "store1";
    String storeName2 = "store2";
    String storeName3 = "store3";
    String storeTopicName1 = ProcessorStateManager.storeChangelogTopic(applicationId, storeName1);
    String storeTopicName2 = ProcessorStateManager.storeChangelogTopic(applicationId, storeName2);
    String storeTopicName3 = ProcessorStateManager.storeChangelogTopic(applicationId, storeName3);
    Map<String, String> storeToChangelogTopic = new HashMap<>();
    storeToChangelogTopic.put(storeName1, storeTopicName1);
    storeToChangelogTopic.put(storeName2, storeTopicName2);
    storeToChangelogTopic.put(storeName3, storeTopicName3);
    OffsetCheckpoint checkpoint = new OffsetCheckpoint(new File(stateDirectory.directoryForTask(taskId), ProcessorStateManager.CHECKPOINT_FILE_NAME));
    checkpoint.write(Collections.singletonMap(new TopicPartition(storeTopicName1, 0), lastCheckpointedOffset));
    TopicPartition partition1 = new TopicPartition(storeTopicName1, 0);
    TopicPartition partition2 = new TopicPartition(storeTopicName2, 0);
    TopicPartition partition3 = new TopicPartition(storeTopicName3, 1);
    MockStateStoreSupplier.MockStateStore store1 = new MockStateStoreSupplier.MockStateStore(storeName1, true);
    MockStateStoreSupplier.MockStateStore store2 = new MockStateStoreSupplier.MockStateStore(storeName2, true);
    MockStateStoreSupplier.MockStateStore store3 = new MockStateStoreSupplier.MockStateStore(storeName3, true);
    // if there is an source partition, inherit the partition id
    Set<TopicPartition> sourcePartitions = Utils.mkSet(new TopicPartition(storeTopicName3, 1));
    // standby
    ProcessorStateManager stateMgr = new ProcessorStateManager(taskId, sourcePartitions, true, stateDirectory, storeToChangelogTopic, changelogReader);
    try {
        stateMgr.register(store1, true, store1.stateRestoreCallback);
        stateMgr.register(store2, true, store2.stateRestoreCallback);
        stateMgr.register(store3, true, store3.stateRestoreCallback);
        Map<TopicPartition, Long> changeLogOffsets = stateMgr.checkpointed();
        assertEquals(3, changeLogOffsets.size());
        assertTrue(changeLogOffsets.containsKey(partition1));
        assertTrue(changeLogOffsets.containsKey(partition2));
        assertTrue(changeLogOffsets.containsKey(partition3));
        assertEquals(lastCheckpointedOffset, (long) changeLogOffsets.get(partition1));
        assertEquals(-1L, (long) changeLogOffsets.get(partition2));
        assertEquals(-1L, (long) changeLogOffsets.get(partition3));
    } finally {
        stateMgr.close(Collections.<TopicPartition, Long>emptyMap());
    }
}

Example 14 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project kafka by apache.

the class StandbyTaskTest method shouldCheckpointStoreOffsetsOnCommit.

@Test
public void shouldCheckpointStoreOffsetsOnCommit() throws Exception {
    consumer.assign(Utils.mkList(ktable));
    final Map<TopicPartition, OffsetAndMetadata> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition(ktable.topic(), ktable.partition()), new OffsetAndMetadata(100L));
    consumer.commitSync(committedOffsets);
    restoreStateConsumer.updatePartitions("ktable1", Utils.mkList(new PartitionInfo("ktable1", 0, Node.noNode(), new Node[0], new Node[0])));
    final TaskId taskId = new TaskId(0, 0);
    final MockTime time = new MockTime();
    final StreamsConfig config = createConfig(baseDir);
    final StandbyTask task = new StandbyTask(taskId, applicationId, ktablePartitions, ktableTopology, consumer, changelogReader, config, null, stateDirectory);
    restoreStateConsumer.assign(new ArrayList<>(task.changeLogPartitions()));
    final byte[] serializedValue = Serdes.Integer().serializer().serialize("", 1);
    task.update(ktable, Collections.singletonList(new ConsumerRecord<>(ktable.topic(), ktable.partition(), 50L, serializedValue, serializedValue)));
    time.sleep(config.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG));
    task.commit();
    final Map<TopicPartition, Long> checkpoint = new OffsetCheckpoint(new File(stateDirectory.directoryForTask(taskId), ProcessorStateManager.CHECKPOINT_FILE_NAME)).read();
    assertThat(checkpoint, equalTo(Collections.singletonMap(ktable, 51L)));
}

Example 15 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project apache-kafka-on-k8s by banzaicloud.

the class ProcessorStateManager method checkpoint.

// write the checkpoint
@Override
public void checkpoint(final Map<TopicPartition, Long> ackedOffsets) {
    checkpointableOffsets.putAll(changelogReader.restoredOffsets());
    for (final StateStore store : stores.values()) {
        final String storeName = store.name();
        // it is persistent AND changelog enabled
        if (store.persistent() && storeToChangelogTopic.containsKey(storeName)) {
            final String changelogTopic = storeToChangelogTopic.get(storeName);
            final TopicPartition topicPartition = new TopicPartition(changelogTopic, getPartition(storeName));
            if (ackedOffsets.containsKey(topicPartition)) {
                // store the last offset + 1 (the log position after restoration)
                checkpointableOffsets.put(topicPartition, ackedOffsets.get(topicPartition) + 1);
            } else if (restoredOffsets.containsKey(topicPartition)) {
                checkpointableOffsets.put(topicPartition, restoredOffsets.get(topicPartition));
            }
        }
    }
    // write the checkpoint file before closing
    if (checkpoint == null) {
        checkpoint = new OffsetCheckpoint(new File(baseDir, CHECKPOINT_FILE_NAME));
    }
    log.trace("Writing checkpoint: {}", checkpointableOffsets);
    try {
        checkpoint.write(checkpointableOffsets);
    } catch (final IOException e) {
        log.warn("Failed to write offset checkpoint file to {}: {}", checkpoint, e);
    }
}