use of org.apache.flink.connector.pulsar.sink.writer.topic.TopicMetadataListener in project flink by apache.
the class PulsarSinkBuilder method setTopics.
/**
* Set a pulsar topic list for flink sink. Some topic may not exist currently, consuming this
* non-existed topic wouldn't throw any exception.
*
* @param topics The topic list you would like to consume message.
* @return this PulsarSinkBuilder.
*/
public PulsarSinkBuilder<IN> setTopics(List<String> topics) {
checkState(metadataListener == null, "setTopics couldn't be set twice.");
// Making sure the topic should be distinct.
List<String> topicSet = distinctTopics(topics);
this.metadataListener = new TopicMetadataListener(topicSet);
return this;
}
use of org.apache.flink.connector.pulsar.sink.writer.topic.TopicMetadataListener in project flink by apache.
the class PulsarSinkBuilder method build.
/**
* Build the {@link PulsarSink}.
*
* @return a PulsarSink with the settings made for this builder.
*/
public PulsarSink<IN> build() {
// Change delivery guarantee.
DeliveryGuarantee deliveryGuarantee = configBuilder.get(PULSAR_WRITE_DELIVERY_GUARANTEE);
if (deliveryGuarantee == DeliveryGuarantee.NONE) {
LOG.warn("You haven't set delivery guarantee or set it to NONE, this would cause data loss. Make sure you have known this shortcoming.");
} else if (deliveryGuarantee == DeliveryGuarantee.EXACTLY_ONCE) {
LOG.info("Exactly once require flink checkpoint and your pulsar cluster should support the transaction.");
configBuilder.override(PULSAR_ENABLE_TRANSACTION, true);
configBuilder.override(PULSAR_SEND_TIMEOUT_MS, 0L);
if (!configBuilder.contains(PULSAR_WRITE_TRANSACTION_TIMEOUT)) {
LOG.warn("The default pulsar transaction timeout is 3 hours, make sure it was greater than your checkpoint interval.");
} else {
Long timeout = configBuilder.get(PULSAR_WRITE_TRANSACTION_TIMEOUT);
LOG.warn("The configured transaction timeout is {} mille seconds, make sure it was greater than your checkpoint interval.", timeout);
}
}
if (!configBuilder.contains(PULSAR_PRODUCER_NAME)) {
LOG.warn("We recommend set a readable producer name through setProducerName(String) in production mode.");
}
checkNotNull(serializationSchema, "serializationSchema must be set.");
if (serializationSchema instanceof PulsarSchemaWrapper && !Boolean.TRUE.equals(configBuilder.get(PULSAR_WRITE_SCHEMA_EVOLUTION))) {
LOG.info("It seems like you want to send message in Pulsar Schema." + " You can enableSchemaEvolution for using this feature." + " We would use Schema.BYTES as the default schema if you don't enable this option.");
}
// Topic metadata listener validation.
if (metadataListener == null) {
if (topicRouter == null) {
throw new NullPointerException("No topic names or custom topic router are provided.");
} else {
LOG.warn("No topic set has been provided, make sure your custom topic router support empty topic set.");
this.metadataListener = new TopicMetadataListener();
}
}
// Topic routing mode validate.
if (topicRoutingMode == null) {
LOG.info("No topic routing mode has been chosen. We use round-robin mode as default.");
this.topicRoutingMode = TopicRoutingMode.ROUND_ROBIN;
}
if (messageDelayer == null) {
this.messageDelayer = MessageDelayer.never();
}
// This is an unmodifiable configuration for Pulsar.
// We don't use Pulsar's built-in configure classes for compatible requirement.
SinkConfiguration sinkConfiguration = configBuilder.build(SINK_CONFIG_VALIDATOR, SinkConfiguration::new);
return new PulsarSink<>(sinkConfiguration, serializationSchema, metadataListener, topicRoutingMode, topicRouter, messageDelayer);
}
use of org.apache.flink.connector.pulsar.sink.writer.topic.TopicMetadataListener in project flink by apache.
the class PulsarWriterTest method writeMessageWithoutGuarantee.
@ParameterizedTest
@EnumSource(value = DeliveryGuarantee.class, names = { "AT_LEAST_ONCE", "NONE" })
void writeMessageWithoutGuarantee(DeliveryGuarantee guarantee) throws Exception {
String topic = randomAlphabetic(10);
operator().createTopic(topic, 8);
SinkConfiguration configuration = sinkConfiguration(guarantee);
PulsarSerializationSchema<String> schema = pulsarSchema(STRING);
TopicMetadataListener listener = new TopicMetadataListener(singletonList(topic));
RoundRobinTopicRouter<String> router = new RoundRobinTopicRouter<>(configuration);
FixedMessageDelayer<String> delayer = MessageDelayer.never();
MockInitContext initContext = new MockInitContext();
PulsarWriter<String> writer = new PulsarWriter<>(configuration, schema, listener, router, delayer, initContext);
writer.flush(false);
writer.prepareCommit();
writer.flush(false);
writer.prepareCommit();
String message = randomAlphabetic(10);
writer.write(message, CONTEXT);
writer.flush(false);
Collection<PulsarCommittable> committables = writer.prepareCommit();
if (guarantee != EXACTLY_ONCE) {
assertThat(committables).isEmpty();
} else {
assertThat(committables).hasSize(1);
PulsarCommittable committable = committables.stream().findFirst().orElseThrow(IllegalArgumentException::new);
TransactionCoordinatorClient coordinatorClient = operator().coordinatorClient();
coordinatorClient.commit(committable.getTxnID());
}
String consumedMessage = operator().receiveMessage(topic, STRING).getValue();
assertEquals(consumedMessage, message);
}
Aggregations