Examples with TopicPartition org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition used on opensource projects

Example 1 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarSourceEnumStateSerializer method deserialize.

@Override
public PulsarSourceEnumState deserialize(int version, byte[] serialized) throws IOException {
    // VERSION 0 deserialization
    try (ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
        DataInputStream in = new DataInputStream(bais)) {
        Set<TopicPartition> partitions = deserializeSet(in, deserializePartition(version));
        Set<PulsarPartitionSplit> splits = deserializeSet(in, deserializeSplit(version));
        Map<Integer, Set<PulsarPartitionSplit>> sharedSplits = deserializeMap(in, DataInput::readInt, i -> deserializeSet(i, deserializeSplit(version)));
        Map<Integer, Set<String>> mapping = deserializeMap(in, DataInput::readInt, i -> deserializeSet(i, DataInput::readUTF));
        boolean initialized = in.readBoolean();
        return new PulsarSourceEnumState(partitions, splits, sharedSplits, mapping, initialized);
    }
}

Example 2 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class SplitsAssignmentState method assignSharedSplits.

/**
 * Every split would be shared among available readers.
 */
private Map<Integer, List<PulsarPartitionSplit>> assignSharedSplits(List<Integer> pendingReaders) {
    Map<Integer, List<PulsarPartitionSplit>> assignMap = new HashMap<>();
    // Drain the splits from share pending list.
    for (Integer reader : pendingReaders) {
        Set<PulsarPartitionSplit> pendingSplits = sharedPendingPartitionSplits.remove(reader);
        if (pendingSplits == null) {
            pendingSplits = new HashSet<>();
        }
        Set<String> assignedSplits = readerAssignedSplits.computeIfAbsent(reader, r -> new HashSet<>());
        for (TopicPartition partition : appendedPartitions) {
            String partitionName = partition.toString();
            if (!assignedSplits.contains(partitionName)) {
                pendingSplits.add(createSplit(partition));
                assignedSplits.add(partitionName);
            }
        }
        if (!pendingSplits.isEmpty()) {
            assignMap.put(reader, new ArrayList<>(pendingSplits));
        }
    }
    return assignMap;
}

Example 3 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarSubscriberTest method topicListSubscriber.

@Test
void topicListSubscriber() {
    operator().createTopic(TOPIC1, NUM_PARTITIONS_PER_TOPIC);
    operator().createTopic(TOPIC2, NUM_PARTITIONS_PER_TOPIC);
    PulsarSubscriber subscriber = getTopicListSubscriber(Arrays.asList(TOPIC1, TOPIC2));
    Set<TopicPartition> topicPartitions = subscriber.getSubscribedTopicPartitions(operator().admin(), new FullRangeGenerator(), NUM_PARALLELISM);
    Set<TopicPartition> expectedPartitions = new HashSet<>();
    for (int i = 0; i < NUM_PARTITIONS_PER_TOPIC; i++) {
        expectedPartitions.add(new TopicPartition(TOPIC1, i, createFullRange()));
        expectedPartitions.add(new TopicPartition(TOPIC2, i, createFullRange()));
    }
    assertEquals(expectedPartitions, topicPartitions);
    operator().deleteTopic(TOPIC1);
    operator().deleteTopic(TOPIC2);
}

Example 4 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarPartitionSplitReaderTestBase method seekStartPositionAndHandleSplit.

private void seekStartPositionAndHandleSplit(PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId, MessageId startPosition) {
    TopicPartition partition = new TopicPartition(topicName, partitionId, createFullRange());
    PulsarPartitionSplit split = new PulsarPartitionSplit(partition, StopCursor.never(), null, null);
    SplitsAddition<PulsarPartitionSplit> addition = new SplitsAddition<>(singletonList(split));
    // create consumer and seek before split changes
    try (Consumer<byte[]> consumer = reader.createPulsarConsumer(partition)) {
        // inclusive messageId
        StartCursor startCursor = StartCursor.fromMessageId(startPosition);
        startCursor.seekPosition(partition.getTopic(), partition.getPartitionId(), consumer);
    } catch (PulsarClientException e) {
        sneakyThrow(e);
    }
    reader.handleSplitsChanges(addition);
}

Example 5 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarOrderedSourceReader method snapshotState.

@Override
public List<PulsarPartitionSplit> snapshotState(long checkpointId) {
    List<PulsarPartitionSplit> splits = super.snapshotState(checkpointId);
    // Perform a snapshot for these splits.
    Map<TopicPartition, MessageId> cursors = cursorsToCommit.computeIfAbsent(checkpointId, id -> new HashMap<>());
    // Put the cursors of the active splits.
    for (PulsarPartitionSplit split : splits) {
        MessageId latestConsumedId = split.getLatestConsumedId();
        if (latestConsumedId != null) {
            cursors.put(split.getPartition(), latestConsumedId);
        }
    }
    // Put cursors of all the finished splits.
    cursors.putAll(cursorsOfFinishedSplits);
    return splits;
}