Search in sources :

Example 91 with Bytes

use of org.apache.kafka.common.utils.Bytes in project kafka by apache.

the class AbstractRocksDBSegmentedBytesStore method fetch.

KeyValueIterator<Bytes, byte[]> fetch(final Bytes key, final long from, final long to, final boolean forward) {
    final List<S> searchSpace = keySchema.segmentsToSearch(segments, from, to, forward);
    final Bytes binaryFrom = keySchema.lowerRangeFixedSize(key, from);
    final Bytes binaryTo = keySchema.upperRangeFixedSize(key, to);
    return new SegmentIterator<>(searchSpace.iterator(), keySchema.hasNextCondition(key, key, from, to), binaryFrom, binaryTo, forward);
}
Also used : Bytes(org.apache.kafka.common.utils.Bytes)

Example 92 with Bytes

use of org.apache.kafka.common.utils.Bytes in project kafka by apache.

the class AbstractMergedSortedCacheStoreIterator method next.

@Override
public KeyValue<K, V> next() {
    if (!hasNext()) {
        throw new NoSuchElementException();
    }
    final Bytes nextCacheKey = cacheIterator.hasNext() ? cacheIterator.peekNextKey() : null;
    final KS nextStoreKey = storeIterator.hasNext() ? storeIterator.peekNextKey() : null;
    if (nextCacheKey == null) {
        return nextStoreValue(nextStoreKey);
    }
    if (nextStoreKey == null) {
        return nextCacheValue(nextCacheKey);
    }
    final int comparison = compare(nextCacheKey, nextStoreKey);
    return chooseNextValue(nextCacheKey, nextStoreKey, comparison);
}
Also used : Bytes(org.apache.kafka.common.utils.Bytes) NoSuchElementException(java.util.NoSuchElementException)

Example 93 with Bytes

use of org.apache.kafka.common.utils.Bytes in project kafka by apache.

the class AbstractMergedSortedCacheStoreIterator method peekNextKey.

@Override
public K peekNextKey() {
    if (!hasNext()) {
        throw new NoSuchElementException();
    }
    final Bytes nextCacheKey = cacheIterator.hasNext() ? cacheIterator.peekNextKey() : null;
    final KS nextStoreKey = storeIterator.hasNext() ? storeIterator.peekNextKey() : null;
    if (nextCacheKey == null) {
        return deserializeStoreKey(nextStoreKey);
    }
    if (nextStoreKey == null) {
        return deserializeCacheKey(nextCacheKey);
    }
    final int comparison = compare(nextCacheKey, nextStoreKey);
    return chooseNextKey(nextCacheKey, nextStoreKey, comparison);
}
Also used : Bytes(org.apache.kafka.common.utils.Bytes) NoSuchElementException(java.util.NoSuchElementException)

Example 94 with Bytes

use of org.apache.kafka.common.utils.Bytes in project kafka by apache.

the class KGroupedTableImplTest method shouldAggregateAndMaterializeResults.

@Test
public void shouldAggregateAndMaterializeResults() {
    builder.table(topic, Consumed.with(Serdes.String(), Serdes.String())).groupBy(MockMapper.selectValueKeyValueMapper(), Grouped.with(Serdes.String(), Serdes.String())).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("aggregate").withValueSerde(Serdes.String()).withKeySerde(Serdes.String()));
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        processData(topic, driver);
        {
            {
                final KeyValueStore<String, String> aggregate = driver.getKeyValueStore("aggregate");
                assertThat(aggregate.get("1"), equalTo("0+1+1+1"));
                assertThat(aggregate.get("2"), equalTo("0+2+2"));
            }
            {
                final KeyValueStore<String, ValueAndTimestamp<String>> aggregate = driver.getTimestampedKeyValueStore("aggregate");
                assertThat(aggregate.get("1"), equalTo(ValueAndTimestamp.make("0+1+1+1", 50L)));
                assertThat(aggregate.get("2"), equalTo(ValueAndTimestamp.make("0+2+2", 60L)));
            }
        }
    }
}
Also used : ValueAndTimestamp(org.apache.kafka.streams.state.ValueAndTimestamp) Bytes(org.apache.kafka.common.utils.Bytes) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) Test(org.junit.Test)

Example 95 with Bytes

use of org.apache.kafka.common.utils.Bytes in project kafka by apache.

the class KGroupedTableImplTest method shouldReduce.

@Test
public void shouldReduce() {
    final KeyValueMapper<String, Number, KeyValue<String, Integer>> intProjection = (key, value) -> KeyValue.pair(key, value.intValue());
    final KTable<String, Integer> reduced = builder.table(topic, Consumed.with(Serdes.String(), Serdes.Double()), Materialized.<String, Double, KeyValueStore<Bytes, byte[]>>as("store").withKeySerde(Serdes.String()).withValueSerde(Serdes.Double())).groupBy(intProjection).reduce(MockReducer.INTEGER_ADDER, MockReducer.INTEGER_SUBTRACTOR, Materialized.as("reduced"));
    final MockApiProcessorSupplier<String, Integer, Void, Void> supplier = getReducedResults(reduced);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        assertReduced(supplier.theCapturedProcessor().lastValueAndTimestampPerKey(), topic, driver);
        assertEquals(reduced.queryableStoreName(), "reduced");
    }
}
Also used : MockInitializer(org.apache.kafka.test.MockInitializer) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) Assert.assertThrows(org.junit.Assert.assertThrows) MockReducer(org.apache.kafka.test.MockReducer) TopologyException(org.apache.kafka.streams.errors.TopologyException) ValueAndTimestamp(org.apache.kafka.streams.state.ValueAndTimestamp) KGroupedTable(org.apache.kafka.streams.kstream.KGroupedTable) MockApiProcessorSupplier(org.apache.kafka.test.MockApiProcessorSupplier) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) Map(java.util.Map) Serdes(org.apache.kafka.common.serialization.Serdes) StringSerializer(org.apache.kafka.common.serialization.StringSerializer) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Before(org.junit.Before) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) MockMapper(org.apache.kafka.test.MockMapper) KTable(org.apache.kafka.streams.kstream.KTable) KeyValueMapper(org.apache.kafka.streams.kstream.KeyValueMapper) Properties(java.util.Properties) DoubleSerializer(org.apache.kafka.common.serialization.DoubleSerializer) Consumed(org.apache.kafka.streams.kstream.Consumed) KeyValue(org.apache.kafka.streams.KeyValue) Test(org.junit.Test) Grouped(org.apache.kafka.streams.kstream.Grouped) MockAggregator(org.apache.kafka.test.MockAggregator) Bytes(org.apache.kafka.common.utils.Bytes) Assert.assertNull(org.junit.Assert.assertNull) Materialized(org.apache.kafka.streams.kstream.Materialized) TestInputTopic(org.apache.kafka.streams.TestInputTopic) StreamsTestUtils(org.apache.kafka.test.StreamsTestUtils) Assert.assertEquals(org.junit.Assert.assertEquals) Bytes(org.apache.kafka.common.utils.Bytes) KeyValue(org.apache.kafka.streams.KeyValue) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) Test(org.junit.Test)

Aggregations

Bytes (org.apache.kafka.common.utils.Bytes)398 Test (org.junit.Test)309 Windowed (org.apache.kafka.streams.kstream.Windowed)84 KeyValue (org.apache.kafka.streams.KeyValue)68 StreamsBuilder (org.apache.kafka.streams.StreamsBuilder)53 SessionWindow (org.apache.kafka.streams.kstream.internals.SessionWindow)50 Properties (java.util.Properties)49 ArrayList (java.util.ArrayList)42 StringSerializer (org.apache.kafka.common.serialization.StringSerializer)38 StreamsConfig (org.apache.kafka.streams.StreamsConfig)35 KeyValueStore (org.apache.kafka.streams.state.KeyValueStore)35 Materialized (org.apache.kafka.streams.kstream.Materialized)33 Serdes (org.apache.kafka.common.serialization.Serdes)32 Metrics (org.apache.kafka.common.metrics.Metrics)29 KafkaStreams (org.apache.kafka.streams.KafkaStreams)28 MockStreamsMetrics (org.apache.kafka.streams.processor.internals.MockStreamsMetrics)27 Consumed (org.apache.kafka.streams.kstream.Consumed)25 KTable (org.apache.kafka.streams.kstream.KTable)23 TopologyTestDriver (org.apache.kafka.streams.TopologyTestDriver)22 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)21