Examples with TransactionCoordinatorClient org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient used on opensource projects

Example 1 with TransactionCoordinatorClient

use of org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient in project flink by apache.

the class PulsarCommitter method commit.

@Override
public void commit(Collection<CommitRequest<PulsarCommittable>> requests) throws IOException, InterruptedException {
    TransactionCoordinatorClient client = transactionCoordinatorClient();
    for (CommitRequest<PulsarCommittable> request : requests) {
        PulsarCommittable committable = request.getCommittable();
        TxnID txnID = committable.getTxnID();
        String topic = committable.getTopic();
        LOG.debug("Start committing the Pulsar transaction {} for topic {}", txnID, topic);
        try {
            client.commit(txnID);
        } catch (TransactionCoordinatorClientException e) {
            // This is a known bug for Pulsar Transaction.
            // We have to use instanceof instead of catching them.
            TransactionCoordinatorClientException ex = PulsarTransactionUtils.unwrap(e);
            if (ex instanceof CoordinatorNotFoundException) {
                LOG.error("We couldn't find the Transaction Coordinator from Pulsar broker {}. " + "Check your broker configuration.", committable, ex);
                request.signalFailedWithKnownReason(ex);
            } else if (ex instanceof InvalidTxnStatusException) {
                LOG.error("Unable to commit transaction ({}) because it's in an invalid state. " + "Most likely the transaction has been aborted for some reason. " + "Please check the Pulsar broker logs for more details.", committable, ex);
                request.signalAlreadyCommitted();
            } else if (ex instanceof TransactionNotFoundException) {
                if (request.getNumberOfRetries() == 0) {
                    LOG.error("Unable to commit transaction ({}) because it's not found on Pulsar broker. " + "Most likely the checkpoint interval exceed the transaction timeout.", committable, ex);
                    request.signalFailedWithKnownReason(ex);
                } else {
                    LOG.warn("We can't find the transaction {} after {} retry committing. " + "This may mean that the transaction have been committed in previous but failed with timeout. " + "So we just mark it as committed.", txnID, request.getNumberOfRetries());
                    request.signalAlreadyCommitted();
                }
            } else if (ex instanceof MetaStoreHandlerNotExistsException) {
                LOG.error("We can't find the meta store handler by the mostSigBits from TxnID {}. " + "Did you change the metadata for topic {}?", committable, TRANSACTION_COORDINATOR_ASSIGN, ex);
                request.signalFailedWithKnownReason(ex);
            } else {
                LOG.error("Encountered retriable exception while committing transaction {} for topic {}.", committable, topic, ex);
                int maxRecommitTimes = sinkConfiguration.getMaxRecommitTimes();
                if (request.getNumberOfRetries() < maxRecommitTimes) {
                    request.retryLater();
                } else {
                    String message = String.format("Failed to commit transaction %s after retrying %d times", txnID, maxRecommitTimes);
                    request.signalFailedWithKnownReason(new FlinkRuntimeException(message, ex));
                }
            }
        } catch (Exception e) {
            LOG.error("Transaction ({}) encountered unknown error and data could be potentially lost.", committable, e);
            request.signalFailedWithUnknownReason(e);
        }
    }
}

Example 2 with TransactionCoordinatorClient

use of org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient in project flink by apache.

the class TopicProducerRegister method abortTransactions.

/**
 * Abort the existed transactions. This method would be used when closing PulsarWriter.
 */
private void abortTransactions() {
    if (transactionRegister.isEmpty()) {
        return;
    }
    TransactionCoordinatorClient coordinatorClient = ((PulsarClientImpl) pulsarClient).getTcClient();
    // This null check is used for making sure transaction is enabled in client.
    checkNotNull(coordinatorClient);
    try (Closer closer = Closer.create()) {
        for (Transaction transaction : transactionRegister.values()) {
            TxnID txnID = transaction.getTxnID();
            closer.register(() -> coordinatorClient.abort(txnID));
        }
        clearTransactions();
    } catch (IOException e) {
        throw new FlinkRuntimeException(e);
    }
}

Example 3 with TransactionCoordinatorClient

use of org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient in project flink by apache.

the class PulsarWriterTest method writeMessageWithoutGuarantee.

@ParameterizedTest
@EnumSource(value = DeliveryGuarantee.class, names = { "AT_LEAST_ONCE", "NONE" })
void writeMessageWithoutGuarantee(DeliveryGuarantee guarantee) throws Exception {
    String topic = randomAlphabetic(10);
    operator().createTopic(topic, 8);
    SinkConfiguration configuration = sinkConfiguration(guarantee);
    PulsarSerializationSchema<String> schema = pulsarSchema(STRING);
    TopicMetadataListener listener = new TopicMetadataListener(singletonList(topic));
    RoundRobinTopicRouter<String> router = new RoundRobinTopicRouter<>(configuration);
    FixedMessageDelayer<String> delayer = MessageDelayer.never();
    MockInitContext initContext = new MockInitContext();
    PulsarWriter<String> writer = new PulsarWriter<>(configuration, schema, listener, router, delayer, initContext);
    writer.flush(false);
    writer.prepareCommit();
    writer.flush(false);
    writer.prepareCommit();
    String message = randomAlphabetic(10);
    writer.write(message, CONTEXT);
    writer.flush(false);
    Collection<PulsarCommittable> committables = writer.prepareCommit();
    if (guarantee != EXACTLY_ONCE) {
        assertThat(committables).isEmpty();
    } else {
        assertThat(committables).hasSize(1);
        PulsarCommittable committable = committables.stream().findFirst().orElseThrow(IllegalArgumentException::new);
        TransactionCoordinatorClient coordinatorClient = operator().coordinatorClient();
        coordinatorClient.commit(committable.getTxnID());
    }
    String consumedMessage = operator().receiveMessage(topic, STRING).getValue();
    assertEquals(consumedMessage, message);
}

Example 4 with TransactionCoordinatorClient

use of org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient in project flink by apache.

the class TopicProducerRegisterTest method createMessageBuilderForSendingMessage.

@ParameterizedTest
@EnumSource(DeliveryGuarantee.class)
void createMessageBuilderForSendingMessage(DeliveryGuarantee deliveryGuarantee) throws IOException {
    String topic = randomAlphabetic(10);
    operator().createTopic(topic, 8);
    SinkConfiguration configuration = sinkConfiguration(deliveryGuarantee);
    TopicProducerRegister register = new TopicProducerRegister(configuration);
    String message = randomAlphabetic(10);
    register.createMessageBuilder(topic, Schema.STRING).value(message).send();
    if (deliveryGuarantee == EXACTLY_ONCE) {
        List<PulsarCommittable> committables = register.prepareCommit();
        for (PulsarCommittable committable : committables) {
            TxnID txnID = committable.getTxnID();
            TransactionCoordinatorClient coordinatorClient = operator().coordinatorClient();
            coordinatorClient.commit(txnID);
        }
    }
    Message<String> receiveMessage = operator().receiveMessage(topic, Schema.STRING);
    assertEquals(receiveMessage.getValue(), message);
}

Example 5 with TransactionCoordinatorClient

use of org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient in project flink by apache.

the class PulsarSourceReaderFactory method create.

@SuppressWarnings("java:S2095")
public static <OUT> SourceReader<OUT, PulsarPartitionSplit> create(SourceReaderContext readerContext, PulsarDeserializationSchema<OUT> deserializationSchema, SourceConfiguration sourceConfiguration) {
    PulsarClient pulsarClient = createClient(sourceConfiguration);
    PulsarAdmin pulsarAdmin = createAdmin(sourceConfiguration);
    // Create a message queue with the predefined source option.
    int queueCapacity = sourceConfiguration.getMessageQueueCapacity();
    FutureCompletingBlockingQueue<RecordsWithSplitIds<PulsarMessage<OUT>>> elementsQueue = new FutureCompletingBlockingQueue<>(queueCapacity);
    // Create different pulsar source reader by subscription type.
    SubscriptionType subscriptionType = sourceConfiguration.getSubscriptionType();
    if (subscriptionType == SubscriptionType.Failover || subscriptionType == SubscriptionType.Exclusive) {
        // Create a ordered split reader supplier.
        Supplier<PulsarOrderedPartitionSplitReader<OUT>> splitReaderSupplier = () -> new PulsarOrderedPartitionSplitReader<>(pulsarClient, pulsarAdmin, sourceConfiguration, deserializationSchema);
        return new PulsarOrderedSourceReader<>(elementsQueue, splitReaderSupplier, readerContext, sourceConfiguration, pulsarClient, pulsarAdmin);
    } else if (subscriptionType == SubscriptionType.Shared || subscriptionType == SubscriptionType.Key_Shared) {
        TransactionCoordinatorClient coordinatorClient = ((PulsarClientImpl) pulsarClient).getTcClient();
        if (coordinatorClient == null && !sourceConfiguration.isEnableAutoAcknowledgeMessage()) {
            throw new IllegalStateException("Transaction is required but didn't enabled");
        }
        Supplier<PulsarUnorderedPartitionSplitReader<OUT>> splitReaderSupplier = () -> new PulsarUnorderedPartitionSplitReader<>(pulsarClient, pulsarAdmin, sourceConfiguration, deserializationSchema, coordinatorClient);
        return new PulsarUnorderedSourceReader<>(elementsQueue, splitReaderSupplier, readerContext, sourceConfiguration, pulsarClient, pulsarAdmin, coordinatorClient);
    } else {
        throw new UnsupportedOperationException("This subscription type is not " + subscriptionType + " supported currently.");
    }
}