Examples with PulsarMessage org.apache.flink.connector.pulsar.source.reader.message.PulsarMessage used on opensource projects

Example 1 with PulsarMessage

use of org.apache.flink.connector.pulsar.source.reader.message.PulsarMessage in project flink by apache.

the class PulsarFetcherManagerBase method addSplits.

/**
 * Override this method for supporting multiple thread fetching, one fetcher thread for one
 * split.
 */
@Override
public void addSplits(List<PulsarPartitionSplit> splitsToAdd) {
    for (PulsarPartitionSplit split : splitsToAdd) {
        SplitFetcher<PulsarMessage<T>, PulsarPartitionSplit> fetcher = getOrCreateFetcher(split.splitId());
        fetcher.addSplits(singletonList(split));
        // This method could be executed multiple times.
        startFetcher(fetcher);
    }
}

Example 2 with PulsarMessage

use of org.apache.flink.connector.pulsar.source.reader.message.PulsarMessage in project flink by apache.

the class PulsarPartitionSplitReaderBase method fetch.

@Override
public RecordsWithSplitIds<PulsarMessage<OUT>> fetch() throws IOException {
    RecordsBySplits.Builder<PulsarMessage<OUT>> builder = new RecordsBySplits.Builder<>();
    // Return when no split registered to this reader.
    if (pulsarConsumer == null || registeredSplit == null) {
        return builder.build();
    }
    // Set wakeup to false for start consuming.
    wakeup.compareAndSet(true, false);
    StopCursor stopCursor = registeredSplit.getStopCursor();
    String splitId = registeredSplit.splitId();
    PulsarMessageCollector<OUT> collector = new PulsarMessageCollector<>(splitId, builder);
    Deadline deadline = Deadline.fromNow(sourceConfiguration.getMaxFetchTime());
    // Consume message from pulsar until it was woke up by flink reader.
    for (int messageNum = 0; messageNum < sourceConfiguration.getMaxFetchRecords() && deadline.hasTimeLeft() && isNotWakeup(); messageNum++) {
        try {
            Duration timeout = deadline.timeLeftIfAny();
            Message<byte[]> message = pollMessage(timeout);
            if (message == null) {
                break;
            }
            // Deserialize message.
            collector.setMessage(message);
            deserializationSchema.deserialize(message, collector);
            // Acknowledge message if need.
            finishedPollMessage(message);
            if (stopCursor.shouldStop(message)) {
                builder.addFinishedSplit(splitId);
                break;
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            break;
        } catch (TimeoutException e) {
            break;
        } catch (ExecutionException e) {
            LOG.error("Error in polling message from pulsar consumer.", e);
            break;
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    return builder.build();
}