Search in sources :

Example 1 with MAX_VALUE

use of java.lang.Long.MAX_VALUE in project kafka by apache.

the class SuppressionIntegrationTest method shouldAllowDisablingChangelog.

@Test
public void shouldAllowDisablingChangelog() {
    final String testId = "-shouldAllowDisablingChangelog";
    final String appId = getClass().getSimpleName().toLowerCase(Locale.getDefault()) + testId;
    final String input = "input" + testId;
    final String outputSuppressed = "output-suppressed" + testId;
    final String outputRaw = "output-raw" + testId;
    cleanStateBeforeTest(CLUSTER, input, outputRaw, outputSuppressed);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> inputStream = builder.stream(input);
    final KTable<String, String> valueCounts = inputStream.groupByKey().aggregate(() -> "()", (key, value, aggregate) -> aggregate + ",(" + key + ": " + value + ")");
    valueCounts.suppress(untilTimeLimit(ofMillis(MAX_VALUE), maxRecords(1L).emitEarlyWhenFull().withLoggingDisabled())).toStream().to(outputSuppressed);
    valueCounts.toStream().to(outputRaw);
    final Properties streamsConfig = getStreamsConfig(appId);
    streamsConfig.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    streamsConfig.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    final KafkaStreams driver = IntegrationTestUtils.getStartedStreams(streamsConfig, builder, true);
    try {
        produceSynchronously(input, asList(new KeyValueTimestamp<>("k1", "v1", scaledTime(0L)), new KeyValueTimestamp<>("k1", "v2", scaledTime(1L)), new KeyValueTimestamp<>("k2", "v1", scaledTime(2L)), new KeyValueTimestamp<>("x", "x", scaledTime(3L))));
        final boolean rawRecords = waitForAnyRecord(outputRaw);
        final boolean suppressedRecords = waitForAnyRecord(outputSuppressed);
        final Set<String> suppressChangeLog = CLUSTER.getAllTopicsInCluster().stream().filter(s -> s.contains("-changelog")).filter(s -> s.contains("KTABLE-SUPPRESS")).collect(Collectors.toSet());
        assertThat(suppressChangeLog, is(empty()));
        assertThat(rawRecords, Matchers.is(true));
        assertThat(suppressedRecords, is(true));
    } finally {
        driver.close();
        quietlyCleanStateAfterTest(CLUSTER, driver);
    }
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) CoreMatchers.is(org.hamcrest.CoreMatchers.is) CoreMatchers.hasItem(org.hamcrest.CoreMatchers.hasItem) Produced(org.apache.kafka.streams.kstream.Produced) MAX_VALUE(java.lang.Long.MAX_VALUE) Utils.mkProperties(org.apache.kafka.common.utils.Utils.mkProperties) ConsumerRecords(org.apache.kafka.clients.consumer.ConsumerRecords) Utils.mkMap(org.apache.kafka.common.utils.Utils.mkMap) StringDeserializer(org.apache.kafka.common.serialization.StringDeserializer) Serde(org.apache.kafka.common.serialization.Serde) Locale(java.util.Locale) Arrays.asList(java.util.Arrays.asList) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) Map(java.util.Map) BufferConfig.maxRecords(org.apache.kafka.streams.kstream.Suppressed.BufferConfig.maxRecords) Serdes(org.apache.kafka.common.serialization.Serdes) StringSerializer(org.apache.kafka.common.serialization.StringSerializer) Consumer(org.apache.kafka.clients.consumer.Consumer) TopicPartition(org.apache.kafka.common.TopicPartition) AfterClass(org.junit.AfterClass) TestUtils(org.apache.kafka.test.TestUtils) KeyValue(org.apache.kafka.streams.KeyValue) Set(java.util.Set) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) Category(org.junit.experimental.categories.Category) AT_LEAST_ONCE(org.apache.kafka.streams.StreamsConfig.AT_LEAST_ONCE) Collectors(java.util.stream.Collectors) Bytes(org.apache.kafka.common.utils.Bytes) IntegrationTestUtils(org.apache.kafka.streams.integration.utils.IntegrationTestUtils) List(java.util.List) Utils.mkEntry(org.apache.kafka.common.utils.Utils.mkEntry) Materialized(org.apache.kafka.streams.kstream.Materialized) Optional(java.util.Optional) Duration.ofMillis(java.time.Duration.ofMillis) IntegrationTestUtils.quietlyCleanStateAfterTest(org.apache.kafka.streams.integration.utils.IntegrationTestUtils.quietlyCleanStateAfterTest) KafkaConsumer(org.apache.kafka.clients.consumer.KafkaConsumer) StreamsConfig(org.apache.kafka.streams.StreamsConfig) BeforeClass(org.junit.BeforeClass) IntegrationTest(org.apache.kafka.test.IntegrationTest) KStream(org.apache.kafka.streams.kstream.KStream) BufferConfig.maxBytes(org.apache.kafka.streams.kstream.Suppressed.BufferConfig.maxBytes) DEFAULT_TIMEOUT(org.apache.kafka.streams.integration.utils.IntegrationTestUtils.DEFAULT_TIMEOUT) EmbeddedKafkaCluster(org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) ProducerConfig(org.apache.kafka.clients.producer.ProducerConfig) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) KTable(org.apache.kafka.streams.kstream.KTable) Matchers.empty(org.hamcrest.Matchers.empty) Properties(java.util.Properties) Consumed(org.apache.kafka.streams.kstream.Consumed) TestUtils.waitForCondition(org.apache.kafka.test.TestUtils.waitForCondition) Matchers(org.hamcrest.Matchers) Test(org.junit.Test) IOException(java.io.IOException) KeyValueTimestamp(org.apache.kafka.streams.KeyValueTimestamp) Grouped(org.apache.kafka.streams.kstream.Grouped) Serializer(org.apache.kafka.common.serialization.Serializer) KafkaStreams(org.apache.kafka.streams.KafkaStreams) IntegrationTestUtils.cleanStateBeforeTest(org.apache.kafka.streams.integration.utils.IntegrationTestUtils.cleanStateBeforeTest) Suppressed.untilTimeLimit(org.apache.kafka.streams.kstream.Suppressed.untilTimeLimit) Collections(java.util.Collections) KafkaStreams(org.apache.kafka.streams.KafkaStreams) Serdes(org.apache.kafka.common.serialization.Serdes) Utils.mkProperties(org.apache.kafka.common.utils.Utils.mkProperties) Properties(java.util.Properties) KeyValueTimestamp(org.apache.kafka.streams.KeyValueTimestamp) IntegrationTestUtils.quietlyCleanStateAfterTest(org.apache.kafka.streams.integration.utils.IntegrationTestUtils.quietlyCleanStateAfterTest) IntegrationTest(org.apache.kafka.test.IntegrationTest) Test(org.junit.Test) IntegrationTestUtils.cleanStateBeforeTest(org.apache.kafka.streams.integration.utils.IntegrationTestUtils.cleanStateBeforeTest)

Aggregations

IOException (java.io.IOException)1 MAX_VALUE (java.lang.Long.MAX_VALUE)1 Duration.ofMillis (java.time.Duration.ofMillis)1 Arrays.asList (java.util.Arrays.asList)1 Collections (java.util.Collections)1 List (java.util.List)1 Locale (java.util.Locale)1 Map (java.util.Map)1 Optional (java.util.Optional)1 Properties (java.util.Properties)1 Set (java.util.Set)1 Collectors (java.util.stream.Collectors)1 Consumer (org.apache.kafka.clients.consumer.Consumer)1 ConsumerConfig (org.apache.kafka.clients.consumer.ConsumerConfig)1 ConsumerRecords (org.apache.kafka.clients.consumer.ConsumerRecords)1 KafkaConsumer (org.apache.kafka.clients.consumer.KafkaConsumer)1 ProducerConfig (org.apache.kafka.clients.producer.ProducerConfig)1 TopicPartition (org.apache.kafka.common.TopicPartition)1 Serde (org.apache.kafka.common.serialization.Serde)1 Serdes (org.apache.kafka.common.serialization.Serdes)1