Examples with TxnID org.apache.pulsar.client.api.transaction.TxnID used on opensource projects

Example 1 with TxnID

use of org.apache.pulsar.client.api.transaction.TxnID in project flink by apache.

the class PulsarCommitter method commit.

@Override
public void commit(Collection<CommitRequest<PulsarCommittable>> requests) throws IOException, InterruptedException {
    TransactionCoordinatorClient client = transactionCoordinatorClient();
    for (CommitRequest<PulsarCommittable> request : requests) {
        PulsarCommittable committable = request.getCommittable();
        TxnID txnID = committable.getTxnID();
        String topic = committable.getTopic();
        LOG.debug("Start committing the Pulsar transaction {} for topic {}", txnID, topic);
        try {
            client.commit(txnID);
        } catch (TransactionCoordinatorClientException e) {
            // This is a known bug for Pulsar Transaction.
            // We have to use instanceof instead of catching them.
            TransactionCoordinatorClientException ex = PulsarTransactionUtils.unwrap(e);
            if (ex instanceof CoordinatorNotFoundException) {
                LOG.error("We couldn't find the Transaction Coordinator from Pulsar broker {}. " + "Check your broker configuration.", committable, ex);
                request.signalFailedWithKnownReason(ex);
            } else if (ex instanceof InvalidTxnStatusException) {
                LOG.error("Unable to commit transaction ({}) because it's in an invalid state. " + "Most likely the transaction has been aborted for some reason. " + "Please check the Pulsar broker logs for more details.", committable, ex);
                request.signalAlreadyCommitted();
            } else if (ex instanceof TransactionNotFoundException) {
                if (request.getNumberOfRetries() == 0) {
                    LOG.error("Unable to commit transaction ({}) because it's not found on Pulsar broker. " + "Most likely the checkpoint interval exceed the transaction timeout.", committable, ex);
                    request.signalFailedWithKnownReason(ex);
                } else {
                    LOG.warn("We can't find the transaction {} after {} retry committing. " + "This may mean that the transaction have been committed in previous but failed with timeout. " + "So we just mark it as committed.", txnID, request.getNumberOfRetries());
                    request.signalAlreadyCommitted();
                }
            } else if (ex instanceof MetaStoreHandlerNotExistsException) {
                LOG.error("We can't find the meta store handler by the mostSigBits from TxnID {}. " + "Did you change the metadata for topic {}?", committable, TRANSACTION_COORDINATOR_ASSIGN, ex);
                request.signalFailedWithKnownReason(ex);
            } else {
                LOG.error("Encountered retriable exception while committing transaction {} for topic {}.", committable, topic, ex);
                int maxRecommitTimes = sinkConfiguration.getMaxRecommitTimes();
                if (request.getNumberOfRetries() < maxRecommitTimes) {
                    request.retryLater();
                } else {
                    String message = String.format("Failed to commit transaction %s after retrying %d times", txnID, maxRecommitTimes);
                    request.signalFailedWithKnownReason(new FlinkRuntimeException(message, ex));
                }
            }
        } catch (Exception e) {
            LOG.error("Transaction ({}) encountered unknown error and data could be potentially lost.", committable, e);
            request.signalFailedWithUnknownReason(e);
        }
    }
}

Example 2 with TxnID

use of org.apache.pulsar.client.api.transaction.TxnID in project flink by apache.

the class TopicProducerRegister method prepareCommit.

/**
 * Convert the transactions into a committable list for Pulsar Committer. The transactions would
 * be removed until Flink triggered a checkpoint.
 */
public List<PulsarCommittable> prepareCommit() {
    List<PulsarCommittable> committables = new ArrayList<>(transactionRegister.size());
    transactionRegister.forEach((topic, transaction) -> {
        TxnID txnID = transaction.getTxnID();
        PulsarCommittable committable = new PulsarCommittable(txnID, topic);
        committables.add(committable);
    });
    clearTransactions();
    return committables;
}

Example 3 with TxnID

use of org.apache.pulsar.client.api.transaction.TxnID in project flink by apache.

the class TopicProducerRegister method abortTransactions.

/**
 * Abort the existed transactions. This method would be used when closing PulsarWriter.
 */
private void abortTransactions() {
    if (transactionRegister.isEmpty()) {
        return;
    }
    TransactionCoordinatorClient coordinatorClient = ((PulsarClientImpl) pulsarClient).getTcClient();
    // This null check is used for making sure transaction is enabled in client.
    checkNotNull(coordinatorClient);
    try (Closer closer = Closer.create()) {
        for (Transaction transaction : transactionRegister.values()) {
            TxnID txnID = transaction.getTxnID();
            closer.register(() -> coordinatorClient.abort(txnID));
        }
        clearTransactions();
    } catch (IOException e) {
        throw new FlinkRuntimeException(e);
    }
}

Example 4 with TxnID

use of org.apache.pulsar.client.api.transaction.TxnID in project flink by apache.

the class PulsarCommittableSerializer method deserialize.

@Override
public PulsarCommittable deserialize(int version, byte[] serialized) throws IOException {
    try (final ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
        final DataInputStream in = new DataInputStream(bais)) {
        long mostSigBits = in.readLong();
        long leastSigBits = in.readLong();
        TxnID txnID = new TxnID(mostSigBits, leastSigBits);
        String topic = in.readUTF();
        return new PulsarCommittable(txnID, topic);
    }
}

Example 5 with TxnID

use of org.apache.pulsar.client.api.transaction.TxnID in project flink by apache.

the class PulsarUnorderedSourceReader method snapshotState.

@Override
public List<PulsarPartitionSplit> snapshotState(long checkpointId) {
    LOG.debug("Trigger the new transaction for downstream readers.");
    List<PulsarPartitionSplit> splits = ((PulsarUnorderedFetcherManager<OUT>) splitFetcherManager).snapshotState(checkpointId);
    if (coordinatorClient != null) {
        // Snapshot the transaction status and commit it after checkpoint finished.
        List<TxnID> txnIDs = transactionsToCommit.computeIfAbsent(checkpointId, id -> new ArrayList<>());
        for (PulsarPartitionSplit split : splits) {
            TxnID uncommittedTransactionId = split.getUncommittedTransactionId();
            if (uncommittedTransactionId != null) {
                txnIDs.add(uncommittedTransactionId);
            }
        }
    }
    return splits;
}