Examples with KeyValue org.apache.kafka.streams.KeyValue used on opensource projects

Example 86 with KeyValue

use of org.apache.kafka.streams.KeyValue in project kafka by apache.

the class RocksDBStoreTest method shouldNotThrowExceptionOnRestoreWhenThereIsPreExistingRocksDbFiles.

@Test
public void shouldNotThrowExceptionOnRestoreWhenThereIsPreExistingRocksDbFiles() {
    rocksDBStore.init((StateStoreContext) context, rocksDBStore);
    rocksDBStore.put(new Bytes("existingKey".getBytes(UTF_8)), "existingValue".getBytes(UTF_8));
    rocksDBStore.flush();
    final List<KeyValue<byte[], byte[]>> restoreBytes = new ArrayList<>();
    final byte[] restoredKey = "restoredKey".getBytes(UTF_8);
    final byte[] restoredValue = "restoredValue".getBytes(UTF_8);
    restoreBytes.add(KeyValue.pair(restoredKey, restoredValue));
    context.restore(DB_NAME, restoreBytes);
    assertThat(stringDeserializer.deserialize(null, rocksDBStore.get(new Bytes(stringSerializer.serialize(null, "restoredKey")))), equalTo("restoredValue"));
}

Example 87 with KeyValue

use of org.apache.kafka.streams.KeyValue in project kafka by apache.

the class RocksDBStoreTest method shouldReturnValueOnRange.

@Test
public void shouldReturnValueOnRange() {
    rocksDBStore.init((StateStoreContext) context, rocksDBStore);
    final KeyValue<String, String> kv0 = new KeyValue<>("0", "zero");
    final KeyValue<String, String> kv1 = new KeyValue<>("1", "one");
    final KeyValue<String, String> kv2 = new KeyValue<>("2", "two");
    rocksDBStore.put(new Bytes(kv0.key.getBytes(UTF_8)), kv0.value.getBytes(UTF_8));
    rocksDBStore.put(new Bytes(kv1.key.getBytes(UTF_8)), kv1.value.getBytes(UTF_8));
    rocksDBStore.put(new Bytes(kv2.key.getBytes(UTF_8)), kv2.value.getBytes(UTF_8));
    final LinkedList<KeyValue<String, String>> expectedContents = new LinkedList<>();
    expectedContents.add(kv0);
    expectedContents.add(kv1);
    try (final KeyValueIterator<Bytes, byte[]> iterator = rocksDBStore.range(null, new Bytes(stringSerializer.serialize(null, "1")))) {
        assertEquals(expectedContents, getDeserializedList(iterator));
    }
}

Example 88 with KeyValue

use of org.apache.kafka.streams.KeyValue in project kafka by apache.

the class PageViewTypedDemo method main.

public static void main(final String[] args) {
    final Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pageview-typed");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, JsonTimestampExtractor.class);
    props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, JSONSerde.class);
    props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, JSONSerde.class);
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000L);
    // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, PageView> views = builder.stream("streams-pageview-input", Consumed.with(Serdes.String(), new JSONSerde<>()));
    final KTable<String, UserProfile> users = builder.table("streams-userprofile-input", Consumed.with(Serdes.String(), new JSONSerde<>()));
    final Duration duration24Hours = Duration.ofHours(24);
    final KStream<WindowedPageViewByRegion, RegionCount> regionCount = views.leftJoin(users, (view, profile) -> {
        final PageViewByRegion viewByRegion = new PageViewByRegion();
        viewByRegion.user = view.user;
        viewByRegion.page = view.page;
        if (profile != null) {
            viewByRegion.region = profile.region;
        } else {
            viewByRegion.region = "UNKNOWN";
        }
        return viewByRegion;
    }).map((user, viewRegion) -> new KeyValue<>(viewRegion.region, viewRegion)).groupByKey(Grouped.with(Serdes.String(), new JSONSerde<>())).windowedBy(TimeWindows.ofSizeAndGrace(Duration.ofDays(7), duration24Hours).advanceBy(Duration.ofSeconds(1))).count().toStream().map((key, value) -> {
        final WindowedPageViewByRegion wViewByRegion = new WindowedPageViewByRegion();
        wViewByRegion.windowStart = key.window().start();
        wViewByRegion.region = key.key();
        final RegionCount rCount = new RegionCount();
        rCount.region = key.key();
        rCount.count = value;
        return new KeyValue<>(wViewByRegion, rCount);
    });
    // write to the result topic
    regionCount.to("streams-pageviewstats-typed-output", Produced.with(new JSONSerde<>(), new JSONSerde<>()));
    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    final CountDownLatch latch = new CountDownLatch(1);
    // attach shutdown handler to catch control-c
    Runtime.getRuntime().addShutdownHook(new Thread("streams-pipe-shutdown-hook") {

        @Override
        public void run() {
            streams.close();
            latch.countDown();
        }
    });
    try {
        streams.start();
        latch.await();
    } catch (final Throwable e) {
        e.printStackTrace();
        System.exit(1);
    }
    System.exit(0);
}

Example 89 with KeyValue

use of org.apache.kafka.streams.KeyValue in project kafka by apache.

the class EosIntegrationTest method shouldNotViolateEosIfOneTaskGetsFencedUsingIsolatedAppInstances.

@Test
public void shouldNotViolateEosIfOneTaskGetsFencedUsingIsolatedAppInstances() throws Exception {
    if (eosConfig.equals(StreamsConfig.AT_LEAST_ONCE))
        return;
    try (final KafkaStreams streams1 = getKafkaStreams("streams1", false, "appDir1", 1, eosConfig, MAX_POLL_INTERVAL_MS);
        final KafkaStreams streams2 = getKafkaStreams("streams2", false, "appDir2", 1, eosConfig, MAX_POLL_INTERVAL_MS)) {
        startKafkaStreamsAndWaitForRunningState(streams1, MAX_WAIT_TIME_MS);
        startKafkaStreamsAndWaitForRunningState(streams2, MAX_WAIT_TIME_MS);
        final List<KeyValue<Long, Long>> committedDataBeforeStall = prepareData(0L, 10L, 0L, 1L);
        final List<KeyValue<Long, Long>> uncommittedDataBeforeStall = prepareData(10L, 15L, 0L, 1L);
        final List<KeyValue<Long, Long>> dataBeforeStall = new ArrayList<>(committedDataBeforeStall.size() + uncommittedDataBeforeStall.size());
        dataBeforeStall.addAll(committedDataBeforeStall);
        dataBeforeStall.addAll(uncommittedDataBeforeStall);
        final List<KeyValue<Long, Long>> dataToTriggerFirstRebalance = prepareData(15L, 20L, 0L, 1L);
        final List<KeyValue<Long, Long>> dataAfterSecondRebalance = prepareData(20L, 30L, 0L, 1L);
        writeInputData(committedDataBeforeStall);
        waitForCondition(() -> commitRequested.get() == 2, MAX_WAIT_TIME_MS, "SteamsTasks did not request commit.");
        // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
        // 
        // p-0: ---> 10 rec + C
        // p-1: ---> 10 rec + C
        final List<KeyValue<Long, Long>> committedRecords = readResult(committedDataBeforeStall.size(), CONSUMER_GROUP_ID);
        checkResultPerKey(committedRecords, committedDataBeforeStall, "The committed records before stall do not match what expected");
        writeInputData(uncommittedDataBeforeStall);
        // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
        // 
        // p-0: ---> 10 rec + C  + 5 rec (pending)
        // p-1: ---> 10 rec + C  + 5 rec (pending)
        final List<KeyValue<Long, Long>> uncommittedRecords = readResult(dataBeforeStall.size(), null);
        checkResultPerKey(uncommittedRecords, dataBeforeStall, "The uncommitted records before stall do not match what expected");
        LOG.info("Injecting Stall");
        stallInjected.set(true);
        writeInputData(dataToTriggerFirstRebalance);
        LOG.info("Input Data Written");
        waitForCondition(() -> stallingHost.get() != null, MAX_WAIT_TIME_MS, "Expected a host to start stalling");
        final String observedStallingHost = stallingHost.get();
        final KafkaStreams stallingInstance;
        final KafkaStreams remainingInstance;
        if ("streams1".equals(observedStallingHost)) {
            stallingInstance = streams1;
            remainingInstance = streams2;
        } else if ("streams2".equals(observedStallingHost)) {
            stallingInstance = streams2;
            remainingInstance = streams1;
        } else {
            throw new IllegalArgumentException("unexpected host name: " + observedStallingHost);
        }
        // the stalling instance won't have an updated view, and it doesn't matter what it thinks
        // the assignment is. We only really care that the remaining instance only sees one host
        // that owns both partitions.
        waitForCondition(() -> stallingInstance.metadataForAllStreamsClients().size() == 2 && remainingInstance.metadataForAllStreamsClients().size() == 1 && remainingInstance.metadataForAllStreamsClients().iterator().next().topicPartitions().size() == 2, MAX_WAIT_TIME_MS, () -> "Should have rebalanced.\n" + "Streams1[" + streams1.metadataForAllStreamsClients() + "]\n" + "Streams2[" + streams2.metadataForAllStreamsClients() + "]");
        // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
        // 
        // p-0: ---> 10 rec + C  + 5 rec + C    + 5 rec + C
        // p-1: ---> 10 rec + C  + 5 rec + C    + 5 rec + C
        final List<KeyValue<Long, Long>> committedRecordsAfterRebalance = readResult(uncommittedDataBeforeStall.size() + dataToTriggerFirstRebalance.size(), CONSUMER_GROUP_ID);
        final List<KeyValue<Long, Long>> expectedCommittedRecordsAfterRebalance = new ArrayList<>(uncommittedDataBeforeStall.size() + dataToTriggerFirstRebalance.size());
        expectedCommittedRecordsAfterRebalance.addAll(uncommittedDataBeforeStall);
        expectedCommittedRecordsAfterRebalance.addAll(dataToTriggerFirstRebalance);
        checkResultPerKey(committedRecordsAfterRebalance, expectedCommittedRecordsAfterRebalance, "The all committed records after rebalance do not match what expected");
        LOG.info("Releasing Stall");
        doStall = false;
        // Once the stalling host rejoins the group, we expect both instances to see both instances.
        // It doesn't really matter what the assignment is, but we might as well also assert that they
        // both see both partitions assigned exactly once
        waitForCondition(() -> streams1.metadataForAllStreamsClients().size() == 2 && streams2.metadataForAllStreamsClients().size() == 2 && streams1.metadataForAllStreamsClients().stream().mapToLong(meta -> meta.topicPartitions().size()).sum() == 2 && streams2.metadataForAllStreamsClients().stream().mapToLong(meta -> meta.topicPartitions().size()).sum() == 2, MAX_WAIT_TIME_MS, () -> "Should have rebalanced.\n" + "Streams1[" + streams1.metadataForAllStreamsClients() + "]\n" + "Streams2[" + streams2.metadataForAllStreamsClients() + "]");
        writeInputData(dataAfterSecondRebalance);
        // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
        // 
        // p-0: ---> 10 rec + C  + 5 rec + C    + 5 rec + C   + 10 rec + C
        // p-1: ---> 10 rec + C  + 5 rec + C    + 5 rec + C   + 10 rec + C
        final List<KeyValue<Long, Long>> allCommittedRecords = readResult(committedDataBeforeStall.size() + uncommittedDataBeforeStall.size() + dataToTriggerFirstRebalance.size() + dataAfterSecondRebalance.size(), CONSUMER_GROUP_ID + "_ALL");
        final int allCommittedRecordsAfterRecoverySize = committedDataBeforeStall.size() + uncommittedDataBeforeStall.size() + dataToTriggerFirstRebalance.size() + dataAfterSecondRebalance.size();
        final List<KeyValue<Long, Long>> allExpectedCommittedRecordsAfterRecovery = new ArrayList<>(allCommittedRecordsAfterRecoverySize);
        allExpectedCommittedRecordsAfterRecovery.addAll(committedDataBeforeStall);
        allExpectedCommittedRecordsAfterRecovery.addAll(uncommittedDataBeforeStall);
        allExpectedCommittedRecordsAfterRecovery.addAll(dataToTriggerFirstRebalance);
        allExpectedCommittedRecordsAfterRecovery.addAll(dataAfterSecondRebalance);
        checkResultPerKey(allCommittedRecords, allExpectedCommittedRecordsAfterRecovery, "The all committed records after recovery do not match what expected");
    }
}

Example 90 with KeyValue

use of org.apache.kafka.streams.KeyValue in project kafka by apache.

the class EosIntegrationTest method shouldNotViolateEosIfOneTaskFails.

@Test
public void shouldNotViolateEosIfOneTaskFails() throws Exception {
    if (eosConfig.equals(StreamsConfig.AT_LEAST_ONCE))
        return;
    try (final KafkaStreams streams = getKafkaStreams("dummy", false, "appDir", 2, eosConfig, MAX_POLL_INTERVAL_MS)) {
        startKafkaStreamsAndWaitForRunningState(streams, MAX_WAIT_TIME_MS);
        final List<KeyValue<Long, Long>> committedDataBeforeFailure = prepareData(0L, 10L, 0L, 1L);
        final List<KeyValue<Long, Long>> uncommittedDataBeforeFailure = prepareData(10L, 15L, 0L, 1L);
        final List<KeyValue<Long, Long>> dataBeforeFailure = new ArrayList<>(committedDataBeforeFailure.size() + uncommittedDataBeforeFailure.size());
        dataBeforeFailure.addAll(committedDataBeforeFailure);
        dataBeforeFailure.addAll(uncommittedDataBeforeFailure);
        final List<KeyValue<Long, Long>> dataAfterFailure = prepareData(15L, 20L, 0L, 1L);
        writeInputData(committedDataBeforeFailure);
        waitForCondition(() -> commitRequested.get() == 2, MAX_WAIT_TIME_MS, "StreamsTasks did not request commit.");
        // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
        // 
        // p-0: ---> 10 rec + C
        // p-1: ---> 10 rec + C
        final List<KeyValue<Long, Long>> committedRecords = readResult(committedDataBeforeFailure.size(), CONSUMER_GROUP_ID);
        checkResultPerKey(committedRecords, committedDataBeforeFailure, "The committed records before failure do not match what expected");
        writeInputData(uncommittedDataBeforeFailure);
        // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
        // 
        // p-0: ---> 10 rec + C  + 5 rec (pending)
        // p-1: ---> 10 rec + C  + 5 rec (pending)
        final List<KeyValue<Long, Long>> uncommittedRecords = readResult(dataBeforeFailure.size(), null);
        checkResultPerKey(uncommittedRecords, dataBeforeFailure, "The uncommitted records before failure do not match what expected");
        errorInjected.set(true);
        writeInputData(dataAfterFailure);
        waitForCondition(() -> uncaughtException != null, MAX_WAIT_TIME_MS, "Should receive uncaught exception from one StreamThread.");
        // expected end state per output partition (C == COMMIT; A == ABORT; ---> indicate the changes):
        // 
        // p-0: ---> 10 rec + C  + 5 rec + C    + 5 rec + C
        // p-1: ---> 10 rec + C  + 5 rec + C    + 5 rec + C
        final List<KeyValue<Long, Long>> allCommittedRecords = readResult(committedDataBeforeFailure.size() + uncommittedDataBeforeFailure.size() + dataAfterFailure.size(), CONSUMER_GROUP_ID + "_ALL");
        final List<KeyValue<Long, Long>> committedRecordsAfterFailure = readResult(uncommittedDataBeforeFailure.size() + dataAfterFailure.size(), CONSUMER_GROUP_ID);
        final int allCommittedRecordsAfterRecoverySize = committedDataBeforeFailure.size() + uncommittedDataBeforeFailure.size() + dataAfterFailure.size();
        final List<KeyValue<Long, Long>> allExpectedCommittedRecordsAfterRecovery = new ArrayList<>(allCommittedRecordsAfterRecoverySize);
        allExpectedCommittedRecordsAfterRecovery.addAll(committedDataBeforeFailure);
        allExpectedCommittedRecordsAfterRecovery.addAll(uncommittedDataBeforeFailure);
        allExpectedCommittedRecordsAfterRecovery.addAll(dataAfterFailure);
        final int committedRecordsAfterRecoverySize = uncommittedDataBeforeFailure.size() + dataAfterFailure.size();
        final List<KeyValue<Long, Long>> expectedCommittedRecordsAfterRecovery = new ArrayList<>(committedRecordsAfterRecoverySize);
        expectedCommittedRecordsAfterRecovery.addAll(uncommittedDataBeforeFailure);
        expectedCommittedRecordsAfterRecovery.addAll(dataAfterFailure);
        checkResultPerKey(allCommittedRecords, allExpectedCommittedRecordsAfterRecovery, "The all committed records after recovery do not match what expected");
        checkResultPerKey(committedRecordsAfterFailure, expectedCommittedRecordsAfterRecovery, "The committed records after recovery do not match what expected");
        assertThat("Should only get one uncaught exception from Streams.", hasUnexpectedError, is(false));
    }
}