Examples with SinkConfiguration org.apache.flink.connector.pulsar.sink.config.SinkConfiguration used on opensource projects

Example 1 with SinkConfiguration

use of org.apache.flink.connector.pulsar.sink.config.SinkConfiguration in project flink by apache.

the class PulsarSinkBuilder method build.

/**
 * Build the {@link PulsarSink}.
 *
 * @return a PulsarSink with the settings made for this builder.
 */
public PulsarSink<IN> build() {
    // Change delivery guarantee.
    DeliveryGuarantee deliveryGuarantee = configBuilder.get(PULSAR_WRITE_DELIVERY_GUARANTEE);
    if (deliveryGuarantee == DeliveryGuarantee.NONE) {
        LOG.warn("You haven't set delivery guarantee or set it to NONE, this would cause data loss. Make sure you have known this shortcoming.");
    } else if (deliveryGuarantee == DeliveryGuarantee.EXACTLY_ONCE) {
        LOG.info("Exactly once require flink checkpoint and your pulsar cluster should support the transaction.");
        configBuilder.override(PULSAR_ENABLE_TRANSACTION, true);
        configBuilder.override(PULSAR_SEND_TIMEOUT_MS, 0L);
        if (!configBuilder.contains(PULSAR_WRITE_TRANSACTION_TIMEOUT)) {
            LOG.warn("The default pulsar transaction timeout is 3 hours, make sure it was greater than your checkpoint interval.");
        } else {
            Long timeout = configBuilder.get(PULSAR_WRITE_TRANSACTION_TIMEOUT);
            LOG.warn("The configured transaction timeout is {} mille seconds, make sure it was greater than your checkpoint interval.", timeout);
        }
    }
    if (!configBuilder.contains(PULSAR_PRODUCER_NAME)) {
        LOG.warn("We recommend set a readable producer name through setProducerName(String) in production mode.");
    }
    checkNotNull(serializationSchema, "serializationSchema must be set.");
    if (serializationSchema instanceof PulsarSchemaWrapper && !Boolean.TRUE.equals(configBuilder.get(PULSAR_WRITE_SCHEMA_EVOLUTION))) {
        LOG.info("It seems like you want to send message in Pulsar Schema." + " You can enableSchemaEvolution for using this feature." + " We would use Schema.BYTES as the default schema if you don't enable this option.");
    }
    // Topic metadata listener validation.
    if (metadataListener == null) {
        if (topicRouter == null) {
            throw new NullPointerException("No topic names or custom topic router are provided.");
        } else {
            LOG.warn("No topic set has been provided, make sure your custom topic router support empty topic set.");
            this.metadataListener = new TopicMetadataListener();
        }
    }
    // Topic routing mode validate.
    if (topicRoutingMode == null) {
        LOG.info("No topic routing mode has been chosen. We use round-robin mode as default.");
        this.topicRoutingMode = TopicRoutingMode.ROUND_ROBIN;
    }
    if (messageDelayer == null) {
        this.messageDelayer = MessageDelayer.never();
    }
    // This is an unmodifiable configuration for Pulsar.
    // We don't use Pulsar's built-in configure classes for compatible requirement.
    SinkConfiguration sinkConfiguration = configBuilder.build(SINK_CONFIG_VALIDATOR, SinkConfiguration::new);
    return new PulsarSink<>(sinkConfiguration, serializationSchema, metadataListener, topicRoutingMode, topicRouter, messageDelayer);
}

Example 2 with SinkConfiguration

use of org.apache.flink.connector.pulsar.sink.config.SinkConfiguration in project flink by apache.

the class TopicProducerRegisterTest method noneAndAtLeastOnceWouldNotCreateTransaction.

@ParameterizedTest
@EnumSource(value = DeliveryGuarantee.class, names = { "AT_LEAST_ONCE", "NONE" })
void noneAndAtLeastOnceWouldNotCreateTransaction(DeliveryGuarantee deliveryGuarantee) {
    String topic = randomAlphabetic(10);
    operator().createTopic(topic, 8);
    SinkConfiguration configuration = sinkConfiguration(deliveryGuarantee);
    TopicProducerRegister register = new TopicProducerRegister(configuration);
    String message = randomAlphabetic(10);
    register.createMessageBuilder(topic, Schema.STRING).value(message).sendAsync();
    List<PulsarCommittable> committables = register.prepareCommit();
    assertThat(committables).isEmpty();
}

Example 3 with SinkConfiguration

use of org.apache.flink.connector.pulsar.sink.config.SinkConfiguration in project flink by apache.

the class TopicMetadataListenerTest method fetchTopicPartitionUpdate.

@Test
void fetchTopicPartitionUpdate() throws Exception {
    String topic = randomAlphabetic(10);
    operator().createTopic(topic, 8);
    long interval = Duration.ofMinutes(20).toMillis();
    TopicMetadataListener listener = new TopicMetadataListener(singletonList(topic));
    SinkConfiguration configuration = sinkConfiguration(interval);
    TestProcessingTimeService timeService = new TestProcessingTimeService();
    timeService.setCurrentTime(System.currentTimeMillis());
    listener.open(configuration, timeService);
    List<String> topics = listener.availableTopics();
    List<String> desiredTopics = topicPartitions(topic, 8);
    assertThat(topics).isEqualTo(desiredTopics);
    // Increase topic partitions and trigger the metadata update logic.
    operator().increaseTopicPartitions(topic, 16);
    timeService.advance(interval);
    topics = listener.availableTopics();
    desiredTopics = topicPartitions(topic, 16);
    assertThat(topics).isEqualTo(desiredTopics);
}

Example 4 with SinkConfiguration

use of org.apache.flink.connector.pulsar.sink.config.SinkConfiguration in project flink by apache.

the class TopicMetadataListenerTest method fetchTopicPartitionInformation.

@Test
void fetchTopicPartitionInformation() {
    String topic = randomAlphabetic(10);
    operator().createTopic(topic, 8);
    TopicMetadataListener listener = new TopicMetadataListener(singletonList(topic));
    SinkConfiguration configuration = sinkConfiguration(Duration.ofMinutes(10).toMillis());
    TestProcessingTimeService timeService = new TestProcessingTimeService();
    List<String> topics = listener.availableTopics();
    assertThat(topics).isEmpty();
    listener.open(configuration, timeService);
    topics = listener.availableTopics();
    List<String> desiredTopics = topicPartitions(topic, 8);
    assertThat(topics).hasSize(8).isEqualTo(desiredTopics);
}

Example 5 with SinkConfiguration

use of org.apache.flink.connector.pulsar.sink.config.SinkConfiguration in project flink by apache.

the class TopicMetadataListenerTest method listenOnPartitions.

@Test
void listenOnPartitions() throws Exception {
    String topic = randomAlphabetic(10);
    operator().createTopic(topic, 6);
    List<String> partitions = topicPartitions(topic, 6);
    TopicMetadataListener listener = new TopicMetadataListener(partitions);
    long interval = Duration.ofMinutes(15).toMillis();
    SinkConfiguration configuration = sinkConfiguration(interval);
    TestProcessingTimeService timeService = new TestProcessingTimeService();
    List<String> topics = listener.availableTopics();
    assertEquals(topics, partitions);
    listener.open(configuration, timeService);
    topics = listener.availableTopics();
    assertEquals(topics, partitions);
    operator().increaseTopicPartitions(topic, 12);
    timeService.advance(interval);
    topics = listener.availableTopics();
    assertEquals(topics, partitions);
}