Examples with KeyValue org.apache.kafka.streams.KeyValue used on opensource projects

Example 6 with KeyValue

use of org.apache.kafka.streams.KeyValue in project kafka by apache.

the class KTableKTableJoinIntegrationTest method beforeTest.

@BeforeClass
public static void beforeTest() throws Exception {
    CLUSTER.createTopic(TABLE_1);
    CLUSTER.createTopic(TABLE_2);
    CLUSTER.createTopic(TABLE_3);
    CLUSTER.createTopic(OUTPUT);
    streamsConfig = new Properties();
    streamsConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    streamsConfig.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    streamsConfig.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    streamsConfig.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    streamsConfig.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
    streamsConfig.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    final Properties producerConfig = new Properties();
    producerConfig.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    producerConfig.put(ProducerConfig.ACKS_CONFIG, "all");
    producerConfig.put(ProducerConfig.RETRIES_CONFIG, 0);
    producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    final List<KeyValue<String, String>> table1 = Arrays.asList(new KeyValue<>("a", "A1"), new KeyValue<>("b", "B1"));
    final List<KeyValue<String, String>> table2 = Arrays.asList(new KeyValue<>("b", "B2"), new KeyValue<>("c", "C2"));
    final List<KeyValue<String, String>> table3 = Arrays.asList(new KeyValue<>("a", "A3"), new KeyValue<>("b", "B3"), new KeyValue<>("c", "C3"));
    // put table 3 first, to make sure data is there when joining T1 with T2
    IntegrationTestUtils.produceKeyValuesSynchronously(TABLE_3, table3, producerConfig, MOCK_TIME);
    IntegrationTestUtils.produceKeyValuesSynchronously(TABLE_1, table1, producerConfig, MOCK_TIME);
    IntegrationTestUtils.produceKeyValuesSynchronously(TABLE_2, table2, producerConfig, MOCK_TIME);
    CONSUMER_CONFIG.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    CONSUMER_CONFIG.put(ConsumerConfig.GROUP_ID_CONFIG, "ktable-ktable-consumer");
    CONSUMER_CONFIG.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    CONSUMER_CONFIG.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
}

Example 7 with KeyValue

use of org.apache.kafka.streams.KeyValue in project kafka by apache.

the class KTableAggregateTest method shouldForwardToCorrectProcessorNodeWhenMultiCacheEvictions.

@Test
public void shouldForwardToCorrectProcessorNodeWhenMultiCacheEvictions() throws Exception {
    final String tableOne = "tableOne";
    final String tableTwo = "tableTwo";
    final KStreamBuilder builder = new KStreamBuilder();
    final String reduceTopic = "TestDriver-reducer-store-repartition";
    final Map<String, Long> reduceResults = new HashMap<>();
    final KTable<String, String> one = builder.table(Serdes.String(), Serdes.String(), tableOne, tableOne);
    final KTable<Long, String> two = builder.table(Serdes.Long(), Serdes.String(), tableTwo, tableTwo);
    final KTable<String, Long> reduce = two.groupBy(new KeyValueMapper<Long, String, KeyValue<String, Long>>() {

        @Override
        public KeyValue<String, Long> apply(final Long key, final String value) {
            return new KeyValue<>(value, key);
        }
    }, Serdes.String(), Serdes.Long()).reduce(new Reducer<Long>() {

        @Override
        public Long apply(final Long value1, final Long value2) {
            return value1 + value2;
        }
    }, new Reducer<Long>() {

        @Override
        public Long apply(final Long value1, final Long value2) {
            return value1 - value2;
        }
    }, "reducer-store");
    reduce.foreach(new ForeachAction<String, Long>() {

        @Override
        public void apply(final String key, final Long value) {
            reduceResults.put(key, value);
        }
    });
    one.leftJoin(reduce, new ValueJoiner<String, Long, String>() {

        @Override
        public String apply(final String value1, final Long value2) {
            return value1 + ":" + value2;
        }
    }).mapValues(new ValueMapper<String, String>() {

        @Override
        public String apply(final String value) {
            return value;
        }
    });
    driver = new KStreamTestDriver(builder, stateDir, 111);
    driver.process(reduceTopic, "1", new Change<>(1L, null));
    driver.process("tableOne", "2", "2");
    // this should trigger eviction on the reducer-store topic
    driver.process(reduceTopic, "2", new Change<>(2L, null));
    // this wont as it is the same value
    driver.process(reduceTopic, "2", new Change<>(2L, null));
    assertEquals(Long.valueOf(2L), reduceResults.get("2"));
    // this will trigger eviction on the tableOne topic
    // that in turn will cause an eviction on reducer-topic. It will flush
    // key 2 as it is the only dirty entry in the cache
    driver.process("tableOne", "1", "5");
    assertEquals(Long.valueOf(4L), reduceResults.get("2"));
}

Example 8 with KeyValue

use of org.apache.kafka.streams.KeyValue in project kafka by apache.

the class CachingSessionStoreTest method shouldFetchAllSessionsWithSameRecordKey.

@Test
public void shouldFetchAllSessionsWithSameRecordKey() throws Exception {
    final List<KeyValue<Windowed<String>, Long>> expected = Arrays.asList(KeyValue.pair(new Windowed<>("a", new SessionWindow(0, 0)), 1L), KeyValue.pair(new Windowed<>("a", new SessionWindow(10, 10)), 2L), KeyValue.pair(new Windowed<>("a", new SessionWindow(100, 100)), 3L), KeyValue.pair(new Windowed<>("a", new SessionWindow(1000, 1000)), 4L));
    for (KeyValue<Windowed<String>, Long> kv : expected) {
        cachingStore.put(kv.key, kv.value);
    }
    // add one that shouldn't appear in the results
    cachingStore.put(new Windowed<>("aa", new SessionWindow(0, 0)), 5L);
    final List<KeyValue<Windowed<String>, Long>> results = toList(cachingStore.fetch("a"));
    assertEquals(expected, results);
}

Example 9 with KeyValue

use of org.apache.kafka.streams.KeyValue in project kafka by apache.

the class RocksDBSessionStoreTest method shouldPutAndFindSessionsInRange.

@Test
public void shouldPutAndFindSessionsInRange() throws Exception {
    final String key = "a";
    final Windowed<String> a1 = new Windowed<>(key, new SessionWindow(10, 10L));
    final Windowed<String> a2 = new Windowed<>(key, new SessionWindow(500L, 1000L));
    sessionStore.put(a1, 1L);
    sessionStore.put(a2, 2L);
    sessionStore.put(new Windowed<>(key, new SessionWindow(1500L, 2000L)), 1L);
    sessionStore.put(new Windowed<>(key, new SessionWindow(2500L, 3000L)), 2L);
    final List<KeyValue<Windowed<String>, Long>> expected = Arrays.asList(KeyValue.pair(a1, 1L), KeyValue.pair(a2, 2L));
    final KeyValueIterator<Windowed<String>, Long> values = sessionStore.findSessions(key, 0, 1000L);
    assertEquals(expected, toList(values));
}

Example 10 with KeyValue

use of org.apache.kafka.streams.KeyValue in project ksql by confluentinc.

the class EndToEndIntegrationTest method shouldSelectAllFromDerivedStream.

@Test
public void shouldSelectAllFromDerivedStream() throws Exception {
    executeStatement("CREATE STREAM pageviews_female" + " AS SELECT %s.userid AS userid, pageid, regionid, gender " + " FROM %s " + " LEFT JOIN %s ON %s.userid = %s.userid" + " WHERE gender = 'FEMALE';", userTable, pageViewStream, userTable, pageViewStream, userTable);
    final QueuedQueryMetadata queryMetadata = executeQuery("SELECT * from pageviews_female;");
    List<KeyValue<String, GenericRow>> results = new ArrayList<>();
    BlockingQueue<KeyValue<String, GenericRow>> rowQueue = queryMetadata.getRowQueue();
    // From the mock data, we expect exactly 3 page views from female users.
    List<String> expectedPages = Arrays.asList("PAGE_2", "PAGE_5", "PAGE_5");
    List<String> expectedUsers = Arrays.asList("USER_2", "USER_0", "USER_2");
    List<String> actualPages = new ArrayList<>();
    List<String> actualUsers = new ArrayList<>();
    TestUtils.waitForCondition(() -> {
        try {
            log.debug("polling from pageviews_female");
            KeyValue<String, GenericRow> nextRow = rowQueue.poll(8000, TimeUnit.MILLISECONDS);
            if (nextRow != null) {
                results.add(nextRow);
            } else {
                // If we didn't receive any records on the output topic for 8 seconds, it probably means that the join
                // failed because the table data wasn't populated when the stream data was consumed. We should just
                // re populate the stream data to try the join again.
                log.warn("repopulating data in {} because the join returned empty results.", pageViewTopic);
                testHarness.publishTestData(pageViewTopic, pageViewDataProvider, System.currentTimeMillis());
            }
        } catch (Exception e) {
            log.error("Got exception when polling from pageviews_female", e);
        }
        return 3 <= results.size();
    }, 30000, "Could not consume any records from " + pageViewTopic + " for 30 seconds");
    for (KeyValue<String, GenericRow> result : results) {
        List<Object> columns = result.value.getColumns();
        log.debug("pageview join: {}", columns);
        assertEquals(6, columns.size());
        String user = (String) columns.get(2);
        actualUsers.add(user);
        String page = (String) columns.get(3);
        actualPages.add(page);
    }
    assertEquals(expectedPages, actualPages);
    assertEquals(expectedUsers, actualUsers);
}