Examples with PulsarCommittable org.apache.flink.connector.pulsar.sink.committer.PulsarCommittable used on opensource projects

Example 1 with PulsarCommittable

use of org.apache.flink.connector.pulsar.sink.committer.PulsarCommittable in project flink by apache.

the class TopicProducerRegister method prepareCommit.

/**
 * Convert the transactions into a committable list for Pulsar Committer. The transactions would
 * be removed until Flink triggered a checkpoint.
 */
public List<PulsarCommittable> prepareCommit() {
    List<PulsarCommittable> committables = new ArrayList<>(transactionRegister.size());
    transactionRegister.forEach((topic, transaction) -> {
        TxnID txnID = transaction.getTxnID();
        PulsarCommittable committable = new PulsarCommittable(txnID, topic);
        committables.add(committable);
    });
    clearTransactions();
    return committables;
}

Example 2 with PulsarCommittable

use of org.apache.flink.connector.pulsar.sink.committer.PulsarCommittable in project flink by apache.

the class TopicProducerRegisterTest method noneAndAtLeastOnceWouldNotCreateTransaction.

@ParameterizedTest
@EnumSource(value = DeliveryGuarantee.class, names = { "AT_LEAST_ONCE", "NONE" })
void noneAndAtLeastOnceWouldNotCreateTransaction(DeliveryGuarantee deliveryGuarantee) {
    String topic = randomAlphabetic(10);
    operator().createTopic(topic, 8);
    SinkConfiguration configuration = sinkConfiguration(deliveryGuarantee);
    TopicProducerRegister register = new TopicProducerRegister(configuration);
    String message = randomAlphabetic(10);
    register.createMessageBuilder(topic, Schema.STRING).value(message).sendAsync();
    List<PulsarCommittable> committables = register.prepareCommit();
    assertThat(committables).isEmpty();
}

Example 3 with PulsarCommittable

use of org.apache.flink.connector.pulsar.sink.committer.PulsarCommittable in project flink by apache.

the class PulsarWriterTest method writeMessageWithoutGuarantee.

@ParameterizedTest
@EnumSource(value = DeliveryGuarantee.class, names = { "AT_LEAST_ONCE", "NONE" })
void writeMessageWithoutGuarantee(DeliveryGuarantee guarantee) throws Exception {
    String topic = randomAlphabetic(10);
    operator().createTopic(topic, 8);
    SinkConfiguration configuration = sinkConfiguration(guarantee);
    PulsarSerializationSchema<String> schema = pulsarSchema(STRING);
    TopicMetadataListener listener = new TopicMetadataListener(singletonList(topic));
    RoundRobinTopicRouter<String> router = new RoundRobinTopicRouter<>(configuration);
    FixedMessageDelayer<String> delayer = MessageDelayer.never();
    MockInitContext initContext = new MockInitContext();
    PulsarWriter<String> writer = new PulsarWriter<>(configuration, schema, listener, router, delayer, initContext);
    writer.flush(false);
    writer.prepareCommit();
    writer.flush(false);
    writer.prepareCommit();
    String message = randomAlphabetic(10);
    writer.write(message, CONTEXT);
    writer.flush(false);
    Collection<PulsarCommittable> committables = writer.prepareCommit();
    if (guarantee != EXACTLY_ONCE) {
        assertThat(committables).isEmpty();
    } else {
        assertThat(committables).hasSize(1);
        PulsarCommittable committable = committables.stream().findFirst().orElseThrow(IllegalArgumentException::new);
        TransactionCoordinatorClient coordinatorClient = operator().coordinatorClient();
        coordinatorClient.commit(committable.getTxnID());
    }
    String consumedMessage = operator().receiveMessage(topic, STRING).getValue();
    assertEquals(consumedMessage, message);
}

Example 4 with PulsarCommittable

use of org.apache.flink.connector.pulsar.sink.committer.PulsarCommittable in project flink by apache.

the class TopicProducerRegisterTest method createMessageBuilderForSendingMessage.

@ParameterizedTest
@EnumSource(DeliveryGuarantee.class)
void createMessageBuilderForSendingMessage(DeliveryGuarantee deliveryGuarantee) throws IOException {
    String topic = randomAlphabetic(10);
    operator().createTopic(topic, 8);
    SinkConfiguration configuration = sinkConfiguration(deliveryGuarantee);
    TopicProducerRegister register = new TopicProducerRegister(configuration);
    String message = randomAlphabetic(10);
    register.createMessageBuilder(topic, Schema.STRING).value(message).send();
    if (deliveryGuarantee == EXACTLY_ONCE) {
        List<PulsarCommittable> committables = register.prepareCommit();
        for (PulsarCommittable committable : committables) {
            TxnID txnID = committable.getTxnID();
            TransactionCoordinatorClient coordinatorClient = operator().coordinatorClient();
            coordinatorClient.commit(txnID);
        }
    }
    Message<String> receiveMessage = operator().receiveMessage(topic, Schema.STRING);
    assertEquals(receiveMessage.getValue(), message);
}