Examples with PulsarMessageCollector org.apache.flink.connector.pulsar.source.reader.message.PulsarMessageCollector used on opensource projects

Example 1 with PulsarMessageCollector

use of org.apache.flink.connector.pulsar.source.reader.message.PulsarMessageCollector in project flink by apache.

the class PulsarPartitionSplitReaderBase method fetch.

@Override
public RecordsWithSplitIds<PulsarMessage<OUT>> fetch() throws IOException {
    RecordsBySplits.Builder<PulsarMessage<OUT>> builder = new RecordsBySplits.Builder<>();
    // Return when no split registered to this reader.
    if (pulsarConsumer == null || registeredSplit == null) {
        return builder.build();
    }
    // Set wakeup to false for start consuming.
    wakeup.compareAndSet(true, false);
    StopCursor stopCursor = registeredSplit.getStopCursor();
    String splitId = registeredSplit.splitId();
    PulsarMessageCollector<OUT> collector = new PulsarMessageCollector<>(splitId, builder);
    Deadline deadline = Deadline.fromNow(sourceConfiguration.getMaxFetchTime());
    // Consume message from pulsar until it was woke up by flink reader.
    for (int messageNum = 0; messageNum < sourceConfiguration.getMaxFetchRecords() && deadline.hasTimeLeft() && isNotWakeup(); messageNum++) {
        try {
            Duration timeout = deadline.timeLeftIfAny();
            Message<byte[]> message = pollMessage(timeout);
            if (message == null) {
                break;
            }
            // Deserialize message.
            collector.setMessage(message);
            deserializationSchema.deserialize(message, collector);
            // Acknowledge message if need.
            finishedPollMessage(message);
            if (stopCursor.shouldStop(message)) {
                builder.addFinishedSplit(splitId);
                break;
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            break;
        } catch (TimeoutException e) {
            break;
        } catch (ExecutionException e) {
            LOG.error("Error in polling message from pulsar consumer.", e);
            break;
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    return builder.build();
}