Examples with StateStoreMetadata org.apache.kafka.streams.processor.internals.ProcessorStateManager.StateStoreMetadata used on opensource projects

Example 6 with StateStoreMetadata

use of org.apache.kafka.streams.processor.internals.ProcessorStateManager.StateStoreMetadata in project kafka by apache.

the class ProcessorStateManagerTest method shouldRestoreNonTimestampedStoreWithNoConverter.

@Test
public void shouldRestoreNonTimestampedStoreWithNoConverter() {
    final ProcessorStateManager stateMgr = getStateManager(Task.TaskType.ACTIVE);
    try {
        stateMgr.registerStore(persistentStore, persistentStore.stateRestoreCallback, null);
        final StateStoreMetadata storeMetadata = stateMgr.storeMetadata(persistentStorePartition);
        assertThat(storeMetadata, notNullValue());
        stateMgr.restore(storeMetadata, singletonList(consumerRecord));
        assertThat(persistentStore.keys.size(), is(1));
        assertTrue(persistentStore.keys.contains(key));
        // we just check non timestamped value length
        assertEquals(9, persistentStore.values.get(0).length);
    } finally {
        stateMgr.close();
    }
}

Example 7 with StateStoreMetadata

use of org.apache.kafka.streams.processor.internals.ProcessorStateManager.StateStoreMetadata in project kafka by apache.

the class ProcessorStateManagerTest method shouldNotWriteCheckpointForNonPersistentStore.

@Test
public void shouldNotWriteCheckpointForNonPersistentStore() throws IOException {
    final ProcessorStateManager stateMgr = getStateManager(Task.TaskType.ACTIVE);
    try {
        stateMgr.registerStore(nonPersistentStore, nonPersistentStore.stateRestoreCallback, null);
        stateMgr.initializeStoreOffsetsFromCheckpoint(true);
        final StateStoreMetadata storeMetadata = stateMgr.storeMetadata(nonPersistentStorePartition);
        assertThat(storeMetadata, notNullValue());
        stateMgr.updateChangelogOffsets(singletonMap(nonPersistentStorePartition, 876L));
        stateMgr.checkpoint();
        final Map<TopicPartition, Long> read = checkpoint.read();
        assertThat(read, equalTo(emptyMap()));
    } finally {
        stateMgr.close();
    }
}

Example 8 with StateStoreMetadata

use of org.apache.kafka.streams.processor.internals.ProcessorStateManager.StateStoreMetadata in project kafka by apache.

the class StoreChangelogReader method restoreChangelog.

/**
 * restore a changelog with its buffered records if there's any; for active changelogs also check if
 * it has completed the restoration and can transit to COMPLETED state and trigger restore callbacks
 */
private boolean restoreChangelog(final ChangelogMetadata changelogMetadata) {
    final ProcessorStateManager stateManager = changelogMetadata.stateManager;
    final StateStoreMetadata storeMetadata = changelogMetadata.storeMetadata;
    final TopicPartition partition = storeMetadata.changelogPartition();
    final String storeName = storeMetadata.store().name();
    final int numRecords = changelogMetadata.bufferedLimitIndex;
    boolean madeProgress = false;
    if (numRecords != 0) {
        madeProgress = true;
        final List<ConsumerRecord<byte[], byte[]>> records = changelogMetadata.bufferedRecords.subList(0, numRecords);
        stateManager.restore(storeMetadata, records);
        // then we can further optimize to save the array-shift but just set array elements to null
        if (numRecords < changelogMetadata.bufferedRecords.size()) {
            records.clear();
        } else {
            changelogMetadata.bufferedRecords.clear();
        }
        final Long currentOffset = storeMetadata.offset();
        log.trace("Restored {} records from changelog {} to store {}, end offset is {}, current offset is {}", partition, storeName, numRecords, recordEndOffset(changelogMetadata.restoreEndOffset), currentOffset);
        changelogMetadata.bufferedLimitIndex = 0;
        changelogMetadata.totalRestored += numRecords;
        // do not trigger restore listener if we are processing standby tasks
        if (changelogMetadata.stateManager.taskType() == Task.TaskType.ACTIVE) {
            try {
                stateRestoreListener.onBatchRestored(partition, storeName, currentOffset, numRecords);
            } catch (final Exception e) {
                throw new StreamsException("State restore listener failed on batch restored", e);
            }
        }
    }
    // we should check even if there's nothing restored, but do not check completed if we are processing standby tasks
    if (changelogMetadata.stateManager.taskType() == Task.TaskType.ACTIVE && hasRestoredToEnd(changelogMetadata)) {
        madeProgress = true;
        log.info("Finished restoring changelog {} to store {} with a total number of {} records", partition, storeName, changelogMetadata.totalRestored);
        changelogMetadata.transitTo(ChangelogState.COMPLETED);
        pauseChangelogsFromRestoreConsumer(Collections.singleton(partition));
        try {
            stateRestoreListener.onRestoreEnd(partition, storeName, changelogMetadata.totalRestored);
        } catch (final Exception e) {
            throw new StreamsException("State restore listener failed on restore completed", e);
        }
    }
    return madeProgress;
}

Example 9 with StateStoreMetadata

use of org.apache.kafka.streams.processor.internals.ProcessorStateManager.StateStoreMetadata in project kafka by apache.

the class StoreChangelogReader method prepareChangelogs.

private void prepareChangelogs(final Set<ChangelogMetadata> newPartitionsToRestore) {
    // separate those who do not have the current offset loaded from checkpoint
    final Set<TopicPartition> newPartitionsWithoutStartOffset = new HashSet<>();
    for (final ChangelogMetadata changelogMetadata : newPartitionsToRestore) {
        final StateStoreMetadata storeMetadata = changelogMetadata.storeMetadata;
        final TopicPartition partition = storeMetadata.changelogPartition();
        final Long currentOffset = storeMetadata.offset();
        final Long endOffset = changelogs.get(partition).restoreEndOffset;
        if (currentOffset != null) {
            // the current offset is the offset of the last record, so we should set the position
            // as that offset + 1 as the "next" record to fetch; seek is not a blocking call so
            // there's nothing to capture
            restoreConsumer.seek(partition, currentOffset + 1);
            log.debug("Start restoring changelog partition {} from current offset {} to end offset {}.", partition, currentOffset, recordEndOffset(endOffset));
        } else {
            log.debug("Start restoring changelog partition {} from the beginning offset to end offset {} " + "since we cannot find current offset.", partition, recordEndOffset(endOffset));
            newPartitionsWithoutStartOffset.add(partition);
        }
    }
    // seek is not a blocking call so there's nothing to capture
    if (!newPartitionsWithoutStartOffset.isEmpty()) {
        restoreConsumer.seekToBeginning(newPartitionsWithoutStartOffset);
    }
    // do not trigger restore listener if we are processing standby tasks
    for (final ChangelogMetadata changelogMetadata : newPartitionsToRestore) {
        if (changelogMetadata.stateManager.taskType() == Task.TaskType.ACTIVE) {
            final StateStoreMetadata storeMetadata = changelogMetadata.storeMetadata;
            final TopicPartition partition = storeMetadata.changelogPartition();
            final String storeName = storeMetadata.store().name();
            long startOffset = 0L;
            try {
                startOffset = restoreConsumer.position(partition);
            } catch (final TimeoutException swallow) {
            // if we cannot find the starting position at the beginning, just use the default 0L
            } catch (final KafkaException e) {
                // execution, hence it is also okay to wrap it as fatal StreamsException
                throw new StreamsException("Restore consumer get unexpected error trying to get the position " + " of " + partition, e);
            }
            try {
                stateRestoreListener.onRestoreStart(partition, storeName, startOffset, changelogMetadata.restoreEndOffset);
            } catch (final Exception e) {
                throw new StreamsException("State restore listener failed on batch restored", e);
            }
        }
    }
}

Example 10 with StateStoreMetadata

use of org.apache.kafka.streams.processor.internals.ProcessorStateManager.StateStoreMetadata in project kafka by apache.

the class StoreChangelogReader method register.

/**
 * Since it is shared for multiple tasks and hence multiple state managers, the registration would take its
 * corresponding state manager as well for restoring.
 */
@Override
public void register(final TopicPartition partition, final ProcessorStateManager stateManager) {
    final StateStoreMetadata storeMetadata = stateManager.storeMetadata(partition);
    if (storeMetadata == null) {
        throw new IllegalStateException("Cannot find the corresponding state store metadata for changelog " + partition);
    }
    final ChangelogMetadata changelogMetadata = new ChangelogMetadata(storeMetadata, stateManager);
    // initializing limit offset to 0L for standby changelog to effectively disable any restoration until it is updated
    if (stateManager.taskType() == Task.TaskType.STANDBY && stateManager.changelogAsSource(partition)) {
        changelogMetadata.restoreEndOffset = 0L;
    }
    if (changelogs.putIfAbsent(partition, changelogMetadata) != null) {
        throw new IllegalStateException("There is already a changelog registered for " + partition + ", this should not happen: " + changelogs);
    }
}