Examples with OffsetCheckpoint org.apache.kafka.streams.state.internals.OffsetCheckpoint used on opensource projects

Example 21 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project kafka by apache.

the class ProcessorStateManagerTest method setup.

@Before
public void setup() {
    baseDir = TestUtils.tempDirectory();
    stateDirectory = new StateDirectory(new StreamsConfig(new Properties() {

        {
            put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
            put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234");
            put(StreamsConfig.STATE_DIR_CONFIG, baseDir.getPath());
        }
    }), new MockTime(), true, true);
    checkpointFile = new File(stateDirectory.getOrCreateDirectoryForTask(taskId), CHECKPOINT_FILE_NAME);
    checkpoint = new OffsetCheckpoint(checkpointFile);
    expect(storeMetadata.changelogPartition()).andReturn(persistentStorePartition).anyTimes();
    expect(storeMetadata.store()).andReturn(store).anyTimes();
    expect(store.name()).andReturn(persistentStoreName).anyTimes();
    replay(storeMetadata, store);
}

Example 22 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project kafka by apache.

the class StandbyTaskEOSIntegrationTest method buildStreamWithDirtyStateDir.

private KafkaStreams buildStreamWithDirtyStateDir(final String stateDirPath, final CountDownLatch recordProcessLatch) throws Exception {
    final StreamsBuilder builder = new StreamsBuilder();
    final TaskId taskId = new TaskId(0, 0);
    final Properties props = props(stateDirPath);
    final StateDirectory stateDirectory = new StateDirectory(new StreamsConfig(props), new MockTime(), true, false);
    new OffsetCheckpoint(new File(stateDirectory.getOrCreateDirectoryForTask(taskId), ".checkpoint")).write(Collections.singletonMap(new TopicPartition("unknown-topic", 0), 5L));
    assertTrue(new File(stateDirectory.getOrCreateDirectoryForTask(taskId), "rocksdb/KSTREAM-AGGREGATE-STATE-STORE-0000000001").mkdirs());
    builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.Integer())).groupByKey().count().toStream().peek((key, value) -> recordProcessLatch.countDown());
    return new KafkaStreams(builder.build(), props);
}

Example 23 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project kafka by apache.

the class GlobalStateManagerImplTest method shouldRestoreRecordsFromCheckpointToHighWatermark.

@Test
public void shouldRestoreRecordsFromCheckpointToHighWatermark() throws IOException {
    initializeConsumer(5, 5, t1);
    final OffsetCheckpoint offsetCheckpoint = new OffsetCheckpoint(new File(stateManager.baseDir(), StateManagerUtil.CHECKPOINT_FILE_NAME));
    offsetCheckpoint.write(Collections.singletonMap(t1, 5L));
    stateManager.initialize();
    stateManager.registerStore(store1, stateRestoreCallback, null);
    assertEquals(5, stateRestoreCallback.restored.size());
}

Example 24 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project kafka by apache.

the class GlobalStateManagerImplTest method writeCheckpoint.

private Map<TopicPartition, Long> writeCheckpoint() throws IOException {
    final OffsetCheckpoint checkpoint = new OffsetCheckpoint(checkpointFile);
    final Map<TopicPartition, Long> expected = Collections.singletonMap(t1, 1L);
    checkpoint.write(expected);
    return expected;
}

Example 25 with OffsetCheckpoint

use of org.apache.kafka.streams.state.internals.OffsetCheckpoint in project kafka by apache.

the class GlobalStateManagerImplTest method shouldThrowStreamsExceptionForOldTopicPartitions.

@Test
public void shouldThrowStreamsExceptionForOldTopicPartitions() throws IOException {
    final HashMap<TopicPartition, Long> expectedOffsets = new HashMap<>();
    expectedOffsets.put(t1, 1L);
    expectedOffsets.put(t2, 1L);
    expectedOffsets.put(t3, 1L);
    expectedOffsets.put(t4, 1L);
    // add an old topic (a topic not associated with any global state store)
    final HashMap<TopicPartition, Long> startOffsets = new HashMap<>(expectedOffsets);
    final TopicPartition tOld = new TopicPartition("oldTopic", 1);
    startOffsets.put(tOld, 1L);
    // start with a checkpoint file will all topic-partitions: expected and old (not
    // associated with any global state store).
    final OffsetCheckpoint checkpoint = new OffsetCheckpoint(checkpointFile);
    checkpoint.write(startOffsets);
    // initialize will throw exception
    final StreamsException e = assertThrows(StreamsException.class, () -> stateManager.initialize());
    assertThat(e.getMessage(), equalTo("Encountered a topic-partition not associated with any global state store"));
}