Search in sources :

Example 1 with RawMessageImpl

use of org.apache.pulsar.client.impl.RawMessageImpl in project incubator-pulsar by apache.

the class CompactedTopicTest method buildCompactedLedger.

/**
 * Build a compacted ledger, and return the id of the ledger, the position of the different
 * entries in the ledger, and a list of gaps, and the entry which should be returned after the gap.
 */
private Triple<Long, List<Pair<MessageIdData, Long>>, List<Pair<MessageIdData, Long>>> buildCompactedLedger(BookKeeper bk, int count) throws Exception {
    LedgerHandle lh = bk.createLedger(1, 1, Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    List<Pair<MessageIdData, Long>> positions = new ArrayList<>();
    List<Pair<MessageIdData, Long>> idsInGaps = new ArrayList<>();
    AtomicLong ledgerIds = new AtomicLong(10L);
    AtomicLong entryIds = new AtomicLong(0L);
    CompletableFuture.allOf(IntStream.range(0, count).mapToObj((i) -> {
        List<MessageIdData> idsInGap = new ArrayList<MessageIdData>();
        if (r.nextInt(10) == 1) {
            long delta = r.nextInt(10) + 1;
            idsInGap.add(MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.get() + 1).build());
            ledgerIds.addAndGet(delta);
            entryIds.set(0);
        }
        long delta = r.nextInt(5);
        if (delta != 0) {
            idsInGap.add(MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.get() + 1).build());
        }
        MessageIdData id = MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.addAndGet(delta + 1)).build();
        @Cleanup RawMessage m = new RawMessageImpl(id, Unpooled.EMPTY_BUFFER);
        CompletableFuture<Void> f = new CompletableFuture<>();
        ByteBuf buffer = m.serialize();
        lh.asyncAddEntry(buffer, (rc, ledger, eid, ctx) -> {
            if (rc != BKException.Code.OK) {
                f.completeExceptionally(BKException.create(rc));
            } else {
                positions.add(Pair.of(id, eid));
                idsInGap.forEach((gid) -> idsInGaps.add(Pair.of(gid, eid)));
                f.complete(null);
            }
        }, null);
        buffer.release();
        return f;
    }).toArray(CompletableFuture[]::new)).get();
    lh.close();
    return Triple.of(lh.getId(), positions, idsInGaps);
}
Also used : IntStream(java.util.stream.IntStream) RawMessageImpl(org.apache.pulsar.client.impl.RawMessageImpl) RawMessage(org.apache.pulsar.client.api.RawMessage) Cleanup(lombok.Cleanup) Random(java.util.Random) CompletableFuture(java.util.concurrent.CompletableFuture) Test(org.testng.annotations.Test) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) AfterMethod(org.testng.annotations.AfterMethod) ArrayList(java.util.ArrayList) Unpooled(io.netty.buffer.Unpooled) Lists(com.google.common.collect.Lists) Pair(org.apache.commons.lang3.tuple.Pair) Assert(org.testng.Assert) ByteBuf(io.netty.buffer.ByteBuf) Triple(org.apache.commons.lang3.tuple.Triple) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) PropertyAdmin(org.apache.pulsar.common.policies.data.PropertyAdmin) BeforeMethod(org.testng.annotations.BeforeMethod) BookKeeper(org.apache.bookkeeper.client.BookKeeper) BKException(org.apache.bookkeeper.client.BKException) Sets(com.google.common.collect.Sets) AtomicLong(java.util.concurrent.atomic.AtomicLong) MockedPulsarServiceBaseTest(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest) List(java.util.List) AsyncLoadingCache(com.github.benmanes.caffeine.cache.AsyncLoadingCache) MessageIdData(org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData) Collections(java.util.Collections) RawMessageImpl(org.apache.pulsar.client.impl.RawMessageImpl) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) MessageIdData(org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData) ArrayList(java.util.ArrayList) ByteBuf(io.netty.buffer.ByteBuf) AtomicLong(java.util.concurrent.atomic.AtomicLong) CompletableFuture(java.util.concurrent.CompletableFuture) ArrayList(java.util.ArrayList) List(java.util.List) RawMessage(org.apache.pulsar.client.api.RawMessage) Pair(org.apache.commons.lang3.tuple.Pair)

Aggregations

AsyncLoadingCache (com.github.benmanes.caffeine.cache.AsyncLoadingCache)1 Lists (com.google.common.collect.Lists)1 Sets (com.google.common.collect.Sets)1 ByteBuf (io.netty.buffer.ByteBuf)1 Unpooled (io.netty.buffer.Unpooled)1 ArrayList (java.util.ArrayList)1 Collections (java.util.Collections)1 List (java.util.List)1 Random (java.util.Random)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 AtomicLong (java.util.concurrent.atomic.AtomicLong)1 IntStream (java.util.stream.IntStream)1 Cleanup (lombok.Cleanup)1 BKException (org.apache.bookkeeper.client.BKException)1 BookKeeper (org.apache.bookkeeper.client.BookKeeper)1 LedgerHandle (org.apache.bookkeeper.client.LedgerHandle)1 PositionImpl (org.apache.bookkeeper.mledger.impl.PositionImpl)1 Pair (org.apache.commons.lang3.tuple.Pair)1 Triple (org.apache.commons.lang3.tuple.Triple)1 MockedPulsarServiceBaseTest (org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest)1