Examples with ReadOnlyCursor org.apache.bookkeeper.mledger.ReadOnlyCursor used on opensource projects

Example 16 with ReadOnlyCursor

use of org.apache.bookkeeper.mledger.ReadOnlyCursor in project incubator-pulsar by apache.

the class ReadOnlyCursorTest method skipAll.

@Test
void skipAll() throws Exception {
    ManagedLedger ledger = factory.open("skip-all", new ManagedLedgerConfig().setMaxEntriesPerLedger(7).setRetentionTime(1, TimeUnit.HOURS));
    int N = 10;
    for (int i = 0; i < N; i++) {
        ledger.addEntry(("entry-" + i).getBytes());
    }
    ReadOnlyCursor cursor = factory.openReadOnlyCursor("skip-all", PositionImpl.EARLIEST, new ManagedLedgerConfig());
    assertEquals(cursor.getNumberOfEntries(), N);
    assertTrue(cursor.hasMoreEntries());
    cursor.skipEntries(N);
    assertEquals(cursor.getNumberOfEntries(), 0);
    assertFalse(cursor.hasMoreEntries());
    cursor.close();
}

Example 17 with ReadOnlyCursor

use of org.apache.bookkeeper.mledger.ReadOnlyCursor in project incubator-pulsar by apache.

the class ReadOnlyCursorTest method empty.

@Test
void empty() throws Exception {
    factory.open("empty", new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS));
    ReadOnlyCursor cursor = factory.openReadOnlyCursor("empty", PositionImpl.EARLIEST, new ManagedLedgerConfig());
    assertEquals(cursor.getNumberOfEntries(), 0);
    assertFalse(cursor.hasMoreEntries());
    cursor.close();
}

Example 18 with ReadOnlyCursor

use of org.apache.bookkeeper.mledger.ReadOnlyCursor in project incubator-pulsar by apache.

the class ReadOnlyCursorTest method simple.

@Test
void simple() throws Exception {
    ManagedLedger ledger = factory.open("simple", new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS));
    int N = 10;
    for (int i = 0; i < N; i++) {
        ledger.addEntry(("entry-" + i).getBytes());
    }
    ReadOnlyCursor cursor = factory.openReadOnlyCursor("simple", PositionImpl.EARLIEST, new ManagedLedgerConfig());
    assertEquals(cursor.getNumberOfEntries(), N);
    assertTrue(cursor.hasMoreEntries());
    List<Entry> entries = cursor.readEntries(N);
    assertEquals(entries.size(), N);
    assertEquals(cursor.getNumberOfEntries(), 0);
    assertFalse(cursor.hasMoreEntries());
    entries.forEach(Entry::release);
    cursor.close();
    // Ensure we can still write to ledger
    for (int i = 0; i < N; i++) {
        ledger.addEntry(("entry-" + i).getBytes());
    }
    // Open a new cursor
    cursor = factory.openReadOnlyCursor("simple", PositionImpl.EARLIEST, new ManagedLedgerConfig());
    assertEquals(cursor.getNumberOfEntries(), 2 * N);
    assertTrue(cursor.hasMoreEntries());
    entries = cursor.readEntries(N);
    assertEquals(entries.size(), N);
    assertEquals(cursor.getNumberOfEntries(), N);
    assertTrue(cursor.hasMoreEntries());
    entries.forEach(Entry::release);
    entries = cursor.readEntries(N);
    assertEquals(entries.size(), N);
    assertEquals(cursor.getNumberOfEntries(), 0);
    assertFalse(cursor.hasMoreEntries());
    entries.forEach(Entry::release);
    cursor.close();
}

Example 19 with ReadOnlyCursor

use of org.apache.bookkeeper.mledger.ReadOnlyCursor in project incubator-pulsar by apache.

the class TransactionProduceTest method produceTest.

// endAction - commit: true, endAction - abort: false
private void produceTest(boolean endAction) throws Exception {
    final String topic = endAction ? PRODUCE_COMMIT_TOPIC : PRODUCE_ABORT_TOPIC;
    PulsarClient pulsarClient = this.pulsarClient;
    Transaction tnx = pulsarClient.newTransaction().withTransactionTimeout(5, TimeUnit.SECONDS).build().get();
    long txnIdMostBits = ((TransactionImpl) tnx).getTxnIdMostBits();
    long txnIdLeastBits = ((TransactionImpl) tnx).getTxnIdLeastBits();
    Assert.assertTrue(txnIdMostBits > -1);
    Assert.assertTrue(txnIdLeastBits > -1);
    @Cleanup Producer<byte[]> outProducer = pulsarClient.newProducer().topic(topic).sendTimeout(0, TimeUnit.SECONDS).enableBatching(false).create();
    int messageCntPerPartition = 3;
    int messageCnt = TOPIC_PARTITION * messageCntPerPartition;
    String content = "Hello Txn - ";
    Set<String> messageSet = new HashSet<>();
    List<CompletableFuture<MessageId>> futureList = new ArrayList<>();
    for (int i = 0; i < messageCnt; i++) {
        String msg = content + i;
        messageSet.add(msg);
        CompletableFuture<MessageId> produceFuture = outProducer.newMessage(tnx).value(msg.getBytes(UTF_8)).sendAsync();
        futureList.add(produceFuture);
    }
    checkMessageId(futureList, true);
    // the target topic hasn't the commit marker before commit
    for (int i = 0; i < TOPIC_PARTITION; i++) {
        ReadOnlyCursor originTopicCursor = getOriginTopicCursor(topic, i);
        Assert.assertNotNull(originTopicCursor);
        log.info("entries count: {}", originTopicCursor.getNumberOfEntries());
        Assert.assertEquals(messageCntPerPartition, originTopicCursor.getNumberOfEntries());
        List<Entry> entries = originTopicCursor.readEntries(messageCnt);
        // check the messages
        for (int j = 0; j < messageCntPerPartition; j++) {
            MessageMetadata messageMetadata = Commands.parseMessageMetadata(entries.get(j).getDataBuffer());
            Assert.assertEquals(messageMetadata.getTxnidMostBits(), txnIdMostBits);
            Assert.assertEquals(messageMetadata.getTxnidLeastBits(), txnIdLeastBits);
            byte[] bytes = new byte[entries.get(j).getDataBuffer().readableBytes()];
            entries.get(j).getDataBuffer().readBytes(bytes);
            System.out.println(new String(bytes));
            Assert.assertTrue(messageSet.remove(new String(bytes)));
        }
        originTopicCursor.close();
    }
    if (endAction) {
        tnx.commit().get();
    } else {
        tnx.abort().get();
    }
    for (int i = 0; i < TOPIC_PARTITION; i++) {
        // the target topic partition received the commit marker
        ReadOnlyCursor originTopicCursor = getOriginTopicCursor(topic, i);
        List<Entry> entries = originTopicCursor.readEntries((int) originTopicCursor.getNumberOfEntries());
        Assert.assertEquals(messageCntPerPartition + 1, entries.size());
        MessageMetadata messageMetadata = Commands.parseMessageMetadata(entries.get(messageCntPerPartition).getDataBuffer());
        if (endAction) {
            Assert.assertEquals(MarkerType.TXN_COMMIT_VALUE, messageMetadata.getMarkerType());
        } else {
            Assert.assertEquals(MarkerType.TXN_ABORT_VALUE, messageMetadata.getMarkerType());
        }
    }
    Assert.assertEquals(0, messageSet.size());
    log.info("produce and {} test finished.", endAction ? "commit" : "abort");
}

Example 20 with ReadOnlyCursor

use of org.apache.bookkeeper.mledger.ReadOnlyCursor in project incubator-pulsar by apache.

the class TestCacheSizeAllocator method cacheSizeAllocatorTest.

@Test(dataProvider = "cacheSizeProvider", timeOut = 1000 * 20)
public void cacheSizeAllocatorTest(long entryQueueSizeBytes) throws Exception {
    TopicName topicName = TopicName.get("public/default/cache-size-" + entryQueueSizeBytes + "test_" + +RandomUtils.nextInt());
    int totalMsgCnt = 1000;
    MessageIdImpl firstMessageId = prepareData(topicName, totalMsgCnt);
    ReadOnlyCursor readOnlyCursor = pulsar.getManagedLedgerFactory().openReadOnlyCursor(topicName.getPersistenceNamingEncoding(), PositionImpl.get(firstMessageId.getLedgerId(), firstMessageId.getEntryId()), new ManagedLedgerConfig());
    readOnlyCursor.skipEntries(totalMsgCnt);
    PositionImpl lastPosition = (PositionImpl) readOnlyCursor.getReadPosition();
    ObjectMapper objectMapper = new ObjectMapper();
    PulsarSplit pulsarSplit = new PulsarSplit(0, "connector-id", topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName(), totalMsgCnt, new String(Schema.BYTES.getSchemaInfo().getSchema()), Schema.BYTES.getSchemaInfo().getType(), firstMessageId.getEntryId(), lastPosition.getEntryId(), firstMessageId.getLedgerId(), lastPosition.getLedgerId(), TupleDomain.all(), objectMapper.writeValueAsString(new HashMap<>()), null);
    List<PulsarColumnHandle> pulsarColumnHandles = TestPulsarConnector.getColumnColumnHandles(topicName, Schema.BYTES.getSchemaInfo(), PulsarColumnHandle.HandleKeyValueType.NONE, true);
    PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig();
    connectorConfig.setMaxSplitQueueSizeBytes(entryQueueSizeBytes);
    ConnectorContext prestoConnectorContext = new TestingConnectorContext();
    PulsarRecordCursor pulsarRecordCursor = new PulsarRecordCursor(pulsarColumnHandles, pulsarSplit, connectorConfig, pulsar.getManagedLedgerFactory(), new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()));
    Class<PulsarRecordCursor> recordCursorClass = PulsarRecordCursor.class;
    Field entryQueueField = recordCursorClass.getDeclaredField("entryQueue");
    entryQueueField.setAccessible(true);
    SpscArrayQueue<Entry> entryQueue = (SpscArrayQueue<Entry>) entryQueueField.get(pulsarRecordCursor);
    Field messageQueueField = recordCursorClass.getDeclaredField("messageQueue");
    messageQueueField.setAccessible(true);
    SpscArrayQueue<RawMessageImpl> messageQueue = (SpscArrayQueue<RawMessageImpl>) messageQueueField.get(pulsarRecordCursor);
    long maxQueueSize = 0;
    if (entryQueueSizeBytes == -1) {
        maxQueueSize = Long.MAX_VALUE;
    } else if (entryQueueSizeBytes == 0) {
        maxQueueSize = 1;
    } else if (entryQueueSizeBytes > 0) {
        maxQueueSize = entryQueueSizeBytes / 2 / singleEntrySize + 1;
    }
    int receiveCnt = 0;
    while (receiveCnt != totalMsgCnt) {
        if (pulsarRecordCursor.advanceNextPosition()) {
            receiveCnt++;
        }
        Assert.assertTrue(entryQueue.size() <= maxQueueSize);
        Assert.assertTrue(messageQueue.size() <= maxQueueSize);
    }
}