Examples with StopCursor org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor used on opensource projects

Example 1 with StopCursor

use of org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor in project flink by apache.

the class PulsarPartitionSplitSerializer method deserializePulsarPartitionSplit.

public PulsarPartitionSplit deserializePulsarPartitionSplit(int version, DataInputStream in) throws IOException {
    // partition
    TopicPartition partition = deserializeTopicPartition(version, in);
    // stopCursor
    StopCursor stopCursor = deserializeObject(in);
    // latestConsumedId
    MessageId latestConsumedId = null;
    if (in.readBoolean()) {
        byte[] messageIdBytes = deserializeBytes(in);
        latestConsumedId = MessageId.fromByteArray(messageIdBytes);
    }
    // uncommittedTransactionId
    TxnID uncommittedTransactionId = null;
    if (in.readBoolean()) {
        long mostSigBits = in.readLong();
        long leastSigBits = in.readLong();
        uncommittedTransactionId = new TxnID(mostSigBits, leastSigBits);
    }
    // Creation
    return new PulsarPartitionSplit(partition, stopCursor, latestConsumedId, uncommittedTransactionId);
}

Example 2 with StopCursor

use of org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor in project flink by apache.

the class PulsarTestCommonUtils method createPartitionSplit.

public static PulsarPartitionSplit createPartitionSplit(String topic, int partitionId, Boundedness boundedness, MessageId latestConsumedId) {
    TopicPartition topicPartition = new TopicPartition(topic, partitionId, TopicRange.createFullRange());
    StopCursor stopCursor = boundedness == Boundedness.BOUNDED ? StopCursor.latest() : StopCursor.never();
    return new PulsarPartitionSplit(topicPartition, stopCursor, latestConsumedId, null);
}

Example 3 with StopCursor

use of org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor in project flink by apache.

the class PulsarPartitionSplitReaderBase method fetch.

@Override
public RecordsWithSplitIds<PulsarMessage<OUT>> fetch() throws IOException {
    RecordsBySplits.Builder<PulsarMessage<OUT>> builder = new RecordsBySplits.Builder<>();
    // Return when no split registered to this reader.
    if (pulsarConsumer == null || registeredSplit == null) {
        return builder.build();
    }
    // Set wakeup to false for start consuming.
    wakeup.compareAndSet(true, false);
    StopCursor stopCursor = registeredSplit.getStopCursor();
    String splitId = registeredSplit.splitId();
    PulsarMessageCollector<OUT> collector = new PulsarMessageCollector<>(splitId, builder);
    Deadline deadline = Deadline.fromNow(sourceConfiguration.getMaxFetchTime());
    // Consume message from pulsar until it was woke up by flink reader.
    for (int messageNum = 0; messageNum < sourceConfiguration.getMaxFetchRecords() && deadline.hasTimeLeft() && isNotWakeup(); messageNum++) {
        try {
            Duration timeout = deadline.timeLeftIfAny();
            Message<byte[]> message = pollMessage(timeout);
            if (message == null) {
                break;
            }
            // Deserialize message.
            collector.setMessage(message);
            deserializationSchema.deserialize(message, collector);
            // Acknowledge message if need.
            finishedPollMessage(message);
            if (stopCursor.shouldStop(message)) {
                builder.addFinishedSplit(splitId);
                break;
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            break;
        } catch (TimeoutException e) {
            break;
        } catch (ExecutionException e) {
            LOG.error("Error in polling message from pulsar consumer.", e);
            break;
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    return builder.build();
}