Examples with TimestampExtractor org.apache.kafka.streams.processor.TimestampExtractor used on opensource projects

Example 1 with TimestampExtractor

use of org.apache.kafka.streams.processor.TimestampExtractor in project kafka-tutorials by confluentinc.

the class ClickEventTimestampExtractorTest method testExtract.

@Test
public void testExtract() {
    TimestampExtractor timestampExtractor = new ClickEventTimestampExtractor();
    long ts = 3333333L;
    Clicks clickEvent = Clicks.newBuilder().setTimestamp(ts).setIp("ip").setUrl("url").build();
    ConsumerRecord<Object, Object> consumerRecord = new ConsumerRecord<>("topic", 1, 1L, clickEvent.getIp(), clickEvent);
    long extracted = timestampExtractor.extract(consumerRecord, -1L);
    assertEquals(ts, extracted);
}

Example 2 with TimestampExtractor

use of org.apache.kafka.streams.processor.TimestampExtractor in project ksql by confluentinc.

the class SourceBuilderV1Test method shouldBuildStreamWithCorrectTimestampExtractor.

@Test
@SuppressWarnings("unchecked")
public void shouldBuildStreamWithCorrectTimestampExtractor() {
    // Given:
    givenUnwindowedSourceStream();
    final ConsumerRecord<Object, Object> record = mock(ConsumerRecord.class);
    when(record.value()).thenReturn(GenericRow.genericRow("123", A_ROWTIME));
    // When:
    streamSource.build(planBuilder, planInfo);
    // Then:
    verify(consumed).withTimestampExtractor(timestampExtractorCaptor.capture());
    final TimestampExtractor extractor = timestampExtractorCaptor.getValue();
    assertThat(extractor.extract(record, 789), is(A_ROWTIME));
}

Example 3 with TimestampExtractor

use of org.apache.kafka.streams.processor.TimestampExtractor in project ksql by confluentinc.

the class TimestampExtractionPolicyFactoryTest method shouldCreateMetadataPolicyWhenTimestampFieldNotProvided.

@Test
public void shouldCreateMetadataPolicyWhenTimestampFieldNotProvided() {
    // When:
    final TimestampExtractionPolicy result = TimestampExtractionPolicyFactory.create(ksqlConfig, schemaBuilder2.build(), Optional.empty());
    // Then:
    assertThat(result, instanceOf(MetadataTimestampExtractionPolicy.class));
    final TimestampExtractor timestampExtractor = result.create(Optional.empty(), true, logger);
    assertThat(timestampExtractor, instanceOf(LoggingTimestampExtractor.class));
    assertThat(((LoggingTimestampExtractor) timestampExtractor).getDelegate(), instanceOf(MetadataTimestampExtractor.class));
}

Example 4 with TimestampExtractor

use of org.apache.kafka.streams.processor.TimestampExtractor in project ksql by confluentinc.

the class SourceBuilderUtils method buildSourceConsumed.

static <K> Consumed<K, GenericRow> buildSourceConsumed(final SourceStep<?> streamSource, final Serde<K> keySerde, final Serde<GenericRow> valueSerde, final Topology.AutoOffsetReset defaultReset, final RuntimeBuildContext buildContext, final ConsumedFactory consumedFactory) {
    final TimestampExtractor timestampExtractor = timestampExtractor(buildContext.getKsqlConfig(), streamSource.getSourceSchema(), streamSource.getTimestampColumn(), streamSource, buildContext);
    final Consumed<K, GenericRow> consumed = consumedFactory.create(keySerde, valueSerde).withTimestampExtractor(timestampExtractor);
    return consumed.withOffsetResetPolicy(getAutoOffsetReset(defaultReset, buildContext));
}

Example 5 with TimestampExtractor

use of org.apache.kafka.streams.processor.TimestampExtractor in project spring-cloud-stream by spring-cloud.

the class StreamToGlobalKTableFunctionTests method testTimeExtractor.

@Test
void testTimeExtractor() throws Exception {
    SpringApplication app = new SpringApplication(OrderEnricherApplication.class);
    app.setWebApplicationType(WebApplicationType.NONE);
    try (ConfigurableApplicationContext context = app.run("--server.port=0", "--spring.jmx.enabled=false", "--spring.cloud.stream.function.definition=forTimeExtractorTest", "--spring.cloud.stream.bindings.forTimeExtractorTest-in-0.destination=orders", "--spring.cloud.stream.bindings.forTimeExtractorTest-in-1.destination=customers", "--spring.cloud.stream.bindings.forTimeExtractorTest-in-2.destination=products", "--spring.cloud.stream.bindings.forTimeExtractorTest-out-0.destination=enriched-order", "--spring.cloud.stream.kafka.streams.bindings.forTimeExtractorTest-in-0.consumer.timestampExtractorBeanName" + "=timestampExtractor", "--spring.cloud.stream.kafka.streams.bindings.forTimeExtractorTest-in-1.consumer.timestampExtractorBeanName" + "=timestampExtractor", "--spring.cloud.stream.kafka.streams.bindings.forTimeExtractorTest-in-2.consumer.timestampExtractorBeanName" + "=timestampExtractor", "--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde" + "=org.apache.kafka.common.serialization.Serdes$StringSerde", "--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde" + "=org.apache.kafka.common.serialization.Serdes$StringSerde", "--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=10000", "--spring.cloud.stream.kafka.streams.bindings.order.consumer.applicationId=" + "testTimeExtractor-abc", "--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) {
        final KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties = context.getBean(KafkaStreamsExtendedBindingProperties.class);
        final Map<String, KafkaStreamsBindingProperties> bindings = kafkaStreamsExtendedBindingProperties.getBindings();
        final KafkaStreamsBindingProperties kafkaStreamsBindingProperties0 = bindings.get("forTimeExtractorTest-in-0");
        final String timestampExtractorBeanName0 = kafkaStreamsBindingProperties0.getConsumer().getTimestampExtractorBeanName();
        final TimestampExtractor timestampExtractor0 = context.getBean(timestampExtractorBeanName0, TimestampExtractor.class);
        assertThat(timestampExtractor0).isNotNull();
        final KafkaStreamsBindingProperties kafkaStreamsBindingProperties1 = bindings.get("forTimeExtractorTest-in-1");
        final String timestampExtractorBeanName1 = kafkaStreamsBindingProperties1.getConsumer().getTimestampExtractorBeanName();
        final TimestampExtractor timestampExtractor1 = context.getBean(timestampExtractorBeanName1, TimestampExtractor.class);
        assertThat(timestampExtractor1).isNotNull();
        final KafkaStreamsBindingProperties kafkaStreamsBindingProperties2 = bindings.get("forTimeExtractorTest-in-2");
        final String timestampExtractorBeanName2 = kafkaStreamsBindingProperties2.getConsumer().getTimestampExtractorBeanName();
        final TimestampExtractor timestampExtractor2 = context.getBean(timestampExtractorBeanName2, TimestampExtractor.class);
        assertThat(timestampExtractor2).isNotNull();
    }
}