Examples with IntRange org.apache.pulsar.common.api.proto.IntRange used on opensource projects

Example 1 with IntRange

use of org.apache.pulsar.common.api.proto.IntRange in project pulsar by apache.

the class HashRangeExclusiveStickyKeyConsumerSelector method validateKeySharedMeta.

private void validateKeySharedMeta(Consumer consumer) throws BrokerServiceException.ConsumerAssignException {
    if (consumer.getKeySharedMeta() == null) {
        throw new BrokerServiceException.ConsumerAssignException("Must specify key shared meta for consumer.");
    }
    List<IntRange> ranges = consumer.getKeySharedMeta().getHashRangesList();
    if (ranges.isEmpty()) {
        throw new BrokerServiceException.ConsumerAssignException("Ranges for KeyShared policy must not be empty.");
    }
    for (IntRange intRange : ranges) {
        if (intRange.getStart() > intRange.getEnd()) {
            throw new BrokerServiceException.ConsumerAssignException("Fixed hash range start > end");
        }
        Map.Entry<Integer, Consumer> ceilingEntry = rangeMap.ceilingEntry(intRange.getStart());
        Map.Entry<Integer, Consumer> floorEntry = rangeMap.floorEntry(intRange.getEnd());
        if (floorEntry != null && floorEntry.getKey() >= intRange.getStart()) {
            throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer " + floorEntry.getValue());
        }
        if (ceilingEntry != null && ceilingEntry.getKey() <= intRange.getEnd()) {
            throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer " + ceilingEntry.getValue());
        }
        if (ceilingEntry != null && floorEntry != null && ceilingEntry.getValue().equals(floorEntry.getValue())) {
            KeySharedMeta keySharedMeta = ceilingEntry.getValue().getKeySharedMeta();
            for (IntRange range : keySharedMeta.getHashRangesList()) {
                int start = Math.max(intRange.getStart(), range.getStart());
                int end = Math.min(intRange.getEnd(), range.getEnd());
                if (end >= start) {
                    throw new BrokerServiceException.ConsumerAssignException("Range conflict with consumer " + ceilingEntry.getValue());
                }
            }
        }
    }
}

Example 2 with IntRange

use of org.apache.pulsar.common.api.proto.IntRange in project pulsar by apache.

the class HashRangeExclusiveStickyKeyConsumerSelectorTest method testMultipleRangeConflict.

@Test
public void testMultipleRangeConflict() throws BrokerServiceException.ConsumerAssignException {
    HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
    Consumer consumer1 = mock(Consumer.class);
    KeySharedMeta keySharedMeta1 = new KeySharedMeta().setKeySharedMode(KeySharedMode.STICKY);
    keySharedMeta1.addHashRange().setStart(2).setEnd(5);
    when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1);
    Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1);
    selector.addConsumer(consumer1);
    Assert.assertEquals(selector.getRangeConsumer().size(), 2);
    final List<List<IntRange>> testRanges = new ArrayList<>();
    testRanges.add(Lists.newArrayList(new IntRange().setStart(2).setEnd(2), new IntRange().setStart(3).setEnd(3), new IntRange().setStart(4).setEnd(5)));
    testRanges.add(Lists.newArrayList(new IntRange().setStart(0).setEnd(0), new IntRange().setStart(1).setEnd(2)));
    for (List<IntRange> testRange : testRanges) {
        Consumer consumer = mock(Consumer.class);
        KeySharedMeta keySharedMeta = new KeySharedMeta().setKeySharedMode(KeySharedMode.STICKY).addAllHashRanges(testRange);
        when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta);
        Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta);
        try {
            selector.addConsumer(consumer);
            Assert.fail("should be failed");
        } catch (BrokerServiceException.ConsumerAssignException ignore) {
        }
        Assert.assertEquals(selector.getRangeConsumer().size(), 2);
    }
}

Example 3 with IntRange

use of org.apache.pulsar.common.api.proto.IntRange in project pulsar by apache.

the class ManagedCursorTest method testBatchIndexDelete.

@Test
public void testBatchIndexDelete() throws ManagedLedgerException, InterruptedException {
    ManagedLedger ledger = factory.open("test_batch_index_delete");
    ManagedCursor cursor = ledger.openCursor("c1");
    final int totalEntries = 100;
    final Position[] positions = new Position[totalEntries];
    for (int i = 0; i < totalEntries; i++) {
        // add entry
        positions[i] = ledger.addEntry(("entry-" + i).getBytes(Encoding));
    }
    assertEquals(cursor.getNumberOfEntries(), totalEntries);
    deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(2).setEnd(4)));
    List<IntRange> deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10);
    Assert.assertEquals(1, deletedIndexes.size());
    Assert.assertEquals(2, deletedIndexes.get(0).getStart());
    Assert.assertEquals(4, deletedIndexes.get(0).getEnd());
    deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(3).setEnd(8)));
    deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10);
    Assert.assertEquals(1, deletedIndexes.size());
    Assert.assertEquals(2, deletedIndexes.get(0).getStart());
    Assert.assertEquals(8, deletedIndexes.get(0).getEnd());
    deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(0)));
    deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10);
    Assert.assertEquals(2, deletedIndexes.size());
    Assert.assertEquals(0, deletedIndexes.get(0).getStart());
    Assert.assertEquals(0, deletedIndexes.get(0).getEnd());
    Assert.assertEquals(2, deletedIndexes.get(1).getStart());
    Assert.assertEquals(8, deletedIndexes.get(1).getEnd());
    deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(1).setEnd(1)));
    deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(9).setEnd(9)));
    deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10);
    Assert.assertNull(deletedIndexes);
    Assert.assertEquals(positions[0], cursor.getMarkDeletedPosition());
    deleteBatchIndex(cursor, positions[1], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(5)));
    cursor.delete(positions[1]);
    deleteBatchIndex(cursor, positions[1], 10, Lists.newArrayList(new IntRange().setStart(6).setEnd(8)));
    deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[1]), 10);
    Assert.assertNull(deletedIndexes);
    deleteBatchIndex(cursor, positions[2], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(5)));
    cursor.markDelete(positions[3]);
    deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[2]), 10);
    Assert.assertNull(deletedIndexes);
    deleteBatchIndex(cursor, positions[3], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(5)));
    cursor.resetCursor(positions[0]);
    deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[3]), 10);
    Assert.assertNull(deletedIndexes);
}

Example 4 with IntRange

use of org.apache.pulsar.common.api.proto.IntRange in project pulsar by apache.

the class Commands method newSubscribe.

public static ByteBuf newSubscribe(String topic, String subscription, long consumerId, long requestId, SubType subType, int priorityLevel, String consumerName, boolean isDurable, MessageIdData startMessageId, Map<String, String> metadata, boolean readCompacted, boolean isReplicated, InitialPosition subscriptionInitialPosition, long startMessageRollbackDurationInSec, SchemaInfo schemaInfo, boolean createTopicIfDoesNotExist, KeySharedPolicy keySharedPolicy, Map<String, String> subscriptionProperties, long consumerEpoch) {
    BaseCommand cmd = localCmd(Type.SUBSCRIBE);
    CommandSubscribe subscribe = cmd.setSubscribe().setTopic(topic).setSubscription(subscription).setSubType(subType).setConsumerId(consumerId).setConsumerName(consumerName).setRequestId(requestId).setPriorityLevel(priorityLevel).setDurable(isDurable).setReadCompacted(readCompacted).setInitialPosition(subscriptionInitialPosition).setReplicateSubscriptionState(isReplicated).setForceTopicCreation(createTopicIfDoesNotExist).setConsumerEpoch(consumerEpoch);
    if (subscriptionProperties != null && !subscriptionProperties.isEmpty()) {
        List<KeyValue> keyValues = new ArrayList<>();
        subscriptionProperties.forEach((key, value) -> {
            KeyValue keyValue = new KeyValue();
            keyValue.setKey(key);
            keyValue.setValue(value);
            keyValues.add(keyValue);
        });
        subscribe.addAllSubscriptionProperties(keyValues);
    }
    if (keySharedPolicy != null) {
        KeySharedMeta keySharedMeta = subscribe.setKeySharedMeta();
        keySharedMeta.setAllowOutOfOrderDelivery(keySharedPolicy.isAllowOutOfOrderDelivery());
        keySharedMeta.setKeySharedMode(convertKeySharedMode(keySharedPolicy.getKeySharedMode()));
        if (keySharedPolicy instanceof KeySharedPolicy.KeySharedPolicySticky) {
            List<Range> ranges = ((KeySharedPolicy.KeySharedPolicySticky) keySharedPolicy).getRanges();
            for (Range range : ranges) {
                IntRange r = keySharedMeta.addHashRange();
                r.setStart(range.getStart());
                r.setEnd(range.getEnd());
            }
        }
    }
    if (startMessageId != null) {
        subscribe.setStartMessageId().copyFrom(startMessageId);
    }
    if (startMessageRollbackDurationInSec > 0) {
        subscribe.setStartMessageRollbackDurationSec(startMessageRollbackDurationInSec);
    }
    if (!metadata.isEmpty()) {
        metadata.entrySet().forEach(e -> subscribe.addMetadata().setKey(e.getKey()).setValue(e.getValue()));
    }
    if (schemaInfo != null) {
        if (subscribe.hasSchema()) {
            throw new IllegalStateException();
        }
        if (subscribe.setSchema().getPropertiesCount() > 0) {
            throw new IllegalStateException();
        }
        convertSchema(schemaInfo, subscribe.setSchema());
    }
    return serializeWithSize(cmd);
}

Example 5 with IntRange

use of org.apache.pulsar.common.api.proto.IntRange in project pulsar by apache.

the class ManagedCursorTest method testBatchIndexesDeletionPersistAndRecover.

@Test
public void testBatchIndexesDeletionPersistAndRecover() throws ManagedLedgerException, InterruptedException {
    ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
    // Make sure the cursor metadata updated by the cursor ledger ID.
    managedLedgerConfig.setMaxUnackedRangesToPersistInZk(-1);
    ManagedLedger ledger = factory.open("test_batch_indexes_deletion_persistent", managedLedgerConfig);
    ManagedCursor cursor = ledger.openCursor("c1");
    final int totalEntries = 100;
    final Position[] positions = new Position[totalEntries];
    for (int i = 0; i < totalEntries; i++) {
        // add entry
        positions[i] = ledger.addEntry(("entry-" + i).getBytes(Encoding));
    }
    assertEquals(cursor.getNumberOfEntries(), totalEntries);
    deleteBatchIndex(cursor, positions[6], 10, Lists.newArrayList(new IntRange().setStart(1).setEnd(3)));
    deleteBatchIndex(cursor, positions[5], 10, Lists.newArrayList(new IntRange().setStart(3).setEnd(6)));
    deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(9)));
    deleteBatchIndex(cursor, positions[1], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(9)));
    deleteBatchIndex(cursor, positions[2], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(9)));
    deleteBatchIndex(cursor, positions[3], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(9)));
    deleteBatchIndex(cursor, positions[4], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(9)));
    cursor.close();
    ledger.close();
    ledger = factory.open("test_batch_indexes_deletion_persistent", managedLedgerConfig);
    cursor = ledger.openCursor("c1");
    List<IntRange> deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[5]), 10);
    Assert.assertEquals(deletedIndexes.size(), 1);
    Assert.assertEquals(deletedIndexes.get(0).getStart(), 3);
    Assert.assertEquals(deletedIndexes.get(0).getEnd(), 6);
    Assert.assertEquals(cursor.getMarkDeletedPosition(), positions[4]);
    deleteBatchIndex(cursor, positions[5], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(9)));
    deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[5]), 10);
    Assert.assertNull(deletedIndexes);
    Assert.assertEquals(cursor.getMarkDeletedPosition(), positions[5]);
    deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[6]), 10);
    Assert.assertEquals(deletedIndexes.size(), 1);
    Assert.assertEquals(deletedIndexes.get(0).getStart(), 1);
    Assert.assertEquals(deletedIndexes.get(0).getEnd(), 3);
}