Examples with TopicPartition org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition used on opensource projects

Example 6 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarOrderedSourceReader method cumulativeAcknowledgmentMessage.

/**
 * Acknowledge the pulsar topic partition cursor by the last consumed message id.
 */
private void cumulativeAcknowledgmentMessage() {
    Map<TopicPartition, MessageId> cursors = new HashMap<>(cursorsOfFinishedSplits);
    // We reuse snapshotState for acquiring a consume status snapshot.
    // So the checkpoint didn't really happen, so we just pass a fake checkpoint id.
    List<PulsarPartitionSplit> splits = super.snapshotState(1L);
    for (PulsarPartitionSplit split : splits) {
        MessageId latestConsumedId = split.getLatestConsumedId();
        if (latestConsumedId != null) {
            cursors.put(split.getPartition(), latestConsumedId);
        }
    }
    try {
        ((PulsarOrderedFetcherManager<OUT>) splitFetcherManager).acknowledgeMessages(cursors);
        // Clean up the finish splits.
        cursorsOfFinishedSplits.keySet().removeAll(cursors.keySet());
    } catch (Exception e) {
        LOG.error("Fail in auto cursor commit.", e);
        cursorCommitThrowable.compareAndSet(null, e);
    }
}

Example 7 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarPartitionSplitSerializer method deserializePulsarPartitionSplit.

public PulsarPartitionSplit deserializePulsarPartitionSplit(int version, DataInputStream in) throws IOException {
    // partition
    TopicPartition partition = deserializeTopicPartition(version, in);
    // stopCursor
    StopCursor stopCursor = deserializeObject(in);
    // latestConsumedId
    MessageId latestConsumedId = null;
    if (in.readBoolean()) {
        byte[] messageIdBytes = deserializeBytes(in);
        latestConsumedId = MessageId.fromByteArray(messageIdBytes);
    }
    // uncommittedTransactionId
    TxnID uncommittedTransactionId = null;
    if (in.readBoolean()) {
        long mostSigBits = in.readLong();
        long leastSigBits = in.readLong();
        uncommittedTransactionId = new TxnID(mostSigBits, leastSigBits);
    }
    // Creation
    return new PulsarPartitionSplit(partition, stopCursor, latestConsumedId, uncommittedTransactionId);
}

Example 8 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarPartitionSplitSerializer method deserializeTopicPartition.

public TopicPartition deserializeTopicPartition(int version, DataInputStream in) throws IOException {
    // VERSION 0 deserialization
    String topic = in.readUTF();
    int partitionId = in.readInt();
    int start = in.readInt();
    int end = in.readInt();
    TopicRange range = new TopicRange(start, end);
    return new TopicPartition(topic, partitionId, range);
}

Example 9 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarSourceEnumStateSerializerTest method serializeAndDeserializePulsarSourceEnumState.

@Test
void serializeAndDeserializePulsarSourceEnumState() throws Exception {
    Set<TopicPartition> partitions = Sets.newHashSet(new TopicPartition(randomAlphabetic(10), 2, new TopicRange(1, 30)), new TopicPartition(randomAlphabetic(10), 1, createFullRange()));
    Set<PulsarPartitionSplit> splits = Collections.singleton(new PulsarPartitionSplit(new TopicPartition(randomAlphabetic(10), 10, createFullRange()), StopCursor.defaultStopCursor()));
    Map<Integer, Set<PulsarPartitionSplit>> shared = Collections.singletonMap(5, splits);
    Map<Integer, Set<String>> mapping = ImmutableMap.of(1, Sets.newHashSet(randomAlphabetic(10), randomAlphabetic(10)), 2, Sets.newHashSet(randomAlphabetic(10), randomAlphabetic(10)));
    PulsarSourceEnumState state = new PulsarSourceEnumState(partitions, splits, shared, mapping, true);
    byte[] bytes = INSTANCE.serialize(state);
    PulsarSourceEnumState state1 = INSTANCE.deserialize(INSTANCE.getVersion(), bytes);
    assertEquals(state.getAppendedPartitions(), state1.getAppendedPartitions());
    assertEquals(state.getPendingPartitionSplits(), state1.getPendingPartitionSplits());
    assertEquals(state.getReaderAssignedSplits(), state1.getReaderAssignedSplits());
    assertEquals(state.isInitialized(), state1.isInitialized());
    assertNotSame(state, state1);
}

Example 10 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarPartitionSplitReaderTestBase method handleSplit.

protected void handleSplit(PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId, MessageId startPosition) {
    TopicPartition partition = new TopicPartition(topicName, partitionId, createFullRange());
    PulsarPartitionSplit split = new PulsarPartitionSplit(partition, StopCursor.never(), startPosition, null);
    SplitsAddition<PulsarPartitionSplit> addition = new SplitsAddition<>(singletonList(split));
    reader.handleSplitsChanges(addition);
}