Examples with PulsarPartitionSplit org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit used on opensource projects

Example 6 with PulsarPartitionSplit

use of org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit in project flink by apache.

the class PulsarOrderedSourceReaderTest method setupSourceReader.

private void setupSourceReader(PulsarSourceReaderBase<Integer> reader, String topicName, int partitionId, Boundedness boundedness) {
    PulsarPartitionSplit split = createPartitionSplit(topicName, partitionId, boundedness);
    reader.addSplits(Collections.singletonList(split));
    reader.notifyNoMoreSplits();
}

Example 7 with PulsarPartitionSplit

use of org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit in project flink by apache.

the class PulsarPartitionSplitReaderTestBase method seekStartPositionAndHandleSplit.

private void seekStartPositionAndHandleSplit(PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId, MessageId startPosition) {
    TopicPartition partition = new TopicPartition(topicName, partitionId, createFullRange());
    PulsarPartitionSplit split = new PulsarPartitionSplit(partition, StopCursor.never(), null, null);
    SplitsAddition<PulsarPartitionSplit> addition = new SplitsAddition<>(singletonList(split));
    // create consumer and seek before split changes
    try (Consumer<byte[]> consumer = reader.createPulsarConsumer(partition)) {
        // inclusive messageId
        StartCursor startCursor = StartCursor.fromMessageId(startPosition);
        startCursor.seekPosition(partition.getTopic(), partition.getPartitionId(), consumer);
    } catch (PulsarClientException e) {
        sneakyThrow(e);
    }
    reader.handleSplitsChanges(addition);
}

Example 8 with PulsarPartitionSplit

use of org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit in project flink by apache.

the class PulsarOrderedSourceReader method snapshotState.

@Override
public List<PulsarPartitionSplit> snapshotState(long checkpointId) {
    List<PulsarPartitionSplit> splits = super.snapshotState(checkpointId);
    // Perform a snapshot for these splits.
    Map<TopicPartition, MessageId> cursors = cursorsToCommit.computeIfAbsent(checkpointId, id -> new HashMap<>());
    // Put the cursors of the active splits.
    for (PulsarPartitionSplit split : splits) {
        MessageId latestConsumedId = split.getLatestConsumedId();
        if (latestConsumedId != null) {
            cursors.put(split.getPartition(), latestConsumedId);
        }
    }
    // Put cursors of all the finished splits.
    cursors.putAll(cursorsOfFinishedSplits);
    return splits;
}

Example 9 with PulsarPartitionSplit

use of org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit in project flink by apache.

the class PulsarOrderedSourceReader method cumulativeAcknowledgmentMessage.

/**
 * Acknowledge the pulsar topic partition cursor by the last consumed message id.
 */
private void cumulativeAcknowledgmentMessage() {
    Map<TopicPartition, MessageId> cursors = new HashMap<>(cursorsOfFinishedSplits);
    // We reuse snapshotState for acquiring a consume status snapshot.
    // So the checkpoint didn't really happen, so we just pass a fake checkpoint id.
    List<PulsarPartitionSplit> splits = super.snapshotState(1L);
    for (PulsarPartitionSplit split : splits) {
        MessageId latestConsumedId = split.getLatestConsumedId();
        if (latestConsumedId != null) {
            cursors.put(split.getPartition(), latestConsumedId);
        }
    }
    try {
        ((PulsarOrderedFetcherManager<OUT>) splitFetcherManager).acknowledgeMessages(cursors);
        // Clean up the finish splits.
        cursorsOfFinishedSplits.keySet().removeAll(cursors.keySet());
    } catch (Exception e) {
        LOG.error("Fail in auto cursor commit.", e);
        cursorCommitThrowable.compareAndSet(null, e);
    }
}

Example 10 with PulsarPartitionSplit

use of org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit in project flink by apache.

the class PulsarUnorderedSourceReader method snapshotState.

@Override
public List<PulsarPartitionSplit> snapshotState(long checkpointId) {
    LOG.debug("Trigger the new transaction for downstream readers.");
    List<PulsarPartitionSplit> splits = ((PulsarUnorderedFetcherManager<OUT>) splitFetcherManager).snapshotState(checkpointId);
    if (coordinatorClient != null) {
        // Snapshot the transaction status and commit it after checkpoint finished.
        List<TxnID> txnIDs = transactionsToCommit.computeIfAbsent(checkpointId, id -> new ArrayList<>());
        for (PulsarPartitionSplit split : splits) {
            TxnID uncommittedTransactionId = split.getUncommittedTransactionId();
            if (uncommittedTransactionId != null) {
                txnIDs.add(uncommittedTransactionId);
            }
        }
    }
    return splits;
}