use of org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit in project flink by apache.
the class PulsarOrderedSourceReaderTest method setupSourceReader.
private void setupSourceReader(PulsarSourceReaderBase<Integer> reader, String topicName, int partitionId, Boundedness boundedness) {
PulsarPartitionSplit split = createPartitionSplit(topicName, partitionId, boundedness);
reader.addSplits(Collections.singletonList(split));
reader.notifyNoMoreSplits();
}
use of org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit in project flink by apache.
the class PulsarPartitionSplitReaderTestBase method seekStartPositionAndHandleSplit.
private void seekStartPositionAndHandleSplit(PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId, MessageId startPosition) {
TopicPartition partition = new TopicPartition(topicName, partitionId, createFullRange());
PulsarPartitionSplit split = new PulsarPartitionSplit(partition, StopCursor.never(), null, null);
SplitsAddition<PulsarPartitionSplit> addition = new SplitsAddition<>(singletonList(split));
// create consumer and seek before split changes
try (Consumer<byte[]> consumer = reader.createPulsarConsumer(partition)) {
// inclusive messageId
StartCursor startCursor = StartCursor.fromMessageId(startPosition);
startCursor.seekPosition(partition.getTopic(), partition.getPartitionId(), consumer);
} catch (PulsarClientException e) {
sneakyThrow(e);
}
reader.handleSplitsChanges(addition);
}
use of org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit in project flink by apache.
the class PulsarOrderedSourceReader method snapshotState.
@Override
public List<PulsarPartitionSplit> snapshotState(long checkpointId) {
List<PulsarPartitionSplit> splits = super.snapshotState(checkpointId);
// Perform a snapshot for these splits.
Map<TopicPartition, MessageId> cursors = cursorsToCommit.computeIfAbsent(checkpointId, id -> new HashMap<>());
// Put the cursors of the active splits.
for (PulsarPartitionSplit split : splits) {
MessageId latestConsumedId = split.getLatestConsumedId();
if (latestConsumedId != null) {
cursors.put(split.getPartition(), latestConsumedId);
}
}
// Put cursors of all the finished splits.
cursors.putAll(cursorsOfFinishedSplits);
return splits;
}
use of org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit in project flink by apache.
the class PulsarOrderedSourceReader method cumulativeAcknowledgmentMessage.
/**
* Acknowledge the pulsar topic partition cursor by the last consumed message id.
*/
private void cumulativeAcknowledgmentMessage() {
Map<TopicPartition, MessageId> cursors = new HashMap<>(cursorsOfFinishedSplits);
// We reuse snapshotState for acquiring a consume status snapshot.
// So the checkpoint didn't really happen, so we just pass a fake checkpoint id.
List<PulsarPartitionSplit> splits = super.snapshotState(1L);
for (PulsarPartitionSplit split : splits) {
MessageId latestConsumedId = split.getLatestConsumedId();
if (latestConsumedId != null) {
cursors.put(split.getPartition(), latestConsumedId);
}
}
try {
((PulsarOrderedFetcherManager<OUT>) splitFetcherManager).acknowledgeMessages(cursors);
// Clean up the finish splits.
cursorsOfFinishedSplits.keySet().removeAll(cursors.keySet());
} catch (Exception e) {
LOG.error("Fail in auto cursor commit.", e);
cursorCommitThrowable.compareAndSet(null, e);
}
}
use of org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit in project flink by apache.
the class PulsarUnorderedSourceReader method snapshotState.
@Override
public List<PulsarPartitionSplit> snapshotState(long checkpointId) {
LOG.debug("Trigger the new transaction for downstream readers.");
List<PulsarPartitionSplit> splits = ((PulsarUnorderedFetcherManager<OUT>) splitFetcherManager).snapshotState(checkpointId);
if (coordinatorClient != null) {
// Snapshot the transaction status and commit it after checkpoint finished.
List<TxnID> txnIDs = transactionsToCommit.computeIfAbsent(checkpointId, id -> new ArrayList<>());
for (PulsarPartitionSplit split : splits) {
TxnID uncommittedTransactionId = split.getUncommittedTransactionId();
if (uncommittedTransactionId != null) {
txnIDs.add(uncommittedTransactionId);
}
}
}
return splits;
}
Aggregations