Examples with GetResult org.apache.pulsar.metadata.api.GetResult used on opensource projects

Example 1 with GetResult

use of org.apache.pulsar.metadata.api.GetResult in project starlight-for-kafka by datastax.

the class MessageFetchContext method handleEntries.

private void handleEntries(final List<Entry> entries, final TopicPartition topicPartition, final FetchRequest.PartitionData partitionData, final KafkaTopicConsumerManager tcm, final ManagedCursor cursor, final AtomicLong cursorOffset, final boolean readCommitted) {
    final long highWatermark = MessageMetadataUtils.getHighWatermark(cursor.getManagedLedger());
    // Add new offset back to TCM after entries are read successfully
    tcm.add(cursorOffset.get(), Pair.of(cursor, cursorOffset.get()));
    PartitionLog partitionLog = requestHandler.getReplicaManager().getPartitionLog(topicPartition, namespacePrefix);
    final long lso = (readCommitted ? partitionLog.firstUndecidedOffset().orElse(highWatermark) : highWatermark);
    List<Entry> committedEntries = entries;
    if (readCommitted) {
        committedEntries = getCommittedEntries(entries, lso);
        if (log.isDebugEnabled()) {
            log.debug("Request {}: read {} entries but only {} entries are committed", header, entries.size(), committedEntries.size());
        }
    } else {
        if (log.isDebugEnabled()) {
            log.debug("Request {}: read {} entries", header, entries.size());
        }
    }
    if (committedEntries.isEmpty()) {
        addErrorPartitionResponse(topicPartition, Errors.NONE);
        return;
    }
    // use compatible magic value by apiVersion
    short apiVersion = header.apiVersion();
    final byte magic;
    if (apiVersion <= 1) {
        magic = RecordBatch.MAGIC_VALUE_V0;
    } else if (apiVersion <= 3) {
        magic = RecordBatch.MAGIC_VALUE_V1;
    } else {
        magic = RecordBatch.CURRENT_MAGIC_VALUE;
    }
    CompletableFuture<String> groupNameFuture = requestHandler.getCurrentConnectedGroup().computeIfAbsent(clientHost, clientHost -> {
        CompletableFuture<String> future = new CompletableFuture<>();
        String groupIdPath = GroupIdUtils.groupIdPathFormat(clientHost, header.clientId());
        requestHandler.getMetadataStore().get(requestHandler.getGroupIdStoredPath() + groupIdPath).thenAccept(getResultOpt -> {
            if (getResultOpt.isPresent()) {
                GetResult getResult = getResultOpt.get();
                future.complete(new String(getResult.getValue() == null ? new byte[0] : getResult.getValue(), StandardCharsets.UTF_8));
            } else {
                future.complete("");
            }
        }).exceptionally(ex -> {
            future.completeExceptionally(ex);
            return null;
        });
        return future;
    });
    // this part is heavyweight, and we should not execute in the ManagedLedger Ordered executor thread
    groupNameFuture.whenCompleteAsync((groupName, ex) -> {
        if (ex != null) {
            log.error("Get groupId failed.", ex);
            groupName = "";
        }
        final long startDecodingEntriesNanos = MathUtils.nowInNano();
        final DecodeResult decodeResult = requestHandler.getEntryFormatter().decode(entries, magic);
        requestHandler.requestStats.getFetchDecodeStats().registerSuccessfulEvent(MathUtils.elapsedNanos(startDecodingEntriesNanos), TimeUnit.NANOSECONDS);
        decodeResults.add(decodeResult);
        final MemoryRecords kafkaRecords = decodeResult.getRecords();
        // collect consumer metrics
        decodeResult.updateConsumerStats(topicPartition, entries.size(), groupName, statsLogger);
        List<FetchResponse.AbortedTransaction> abortedTransactions;
        if (readCommitted) {
            abortedTransactions = partitionLog.getAbortedIndexList(partitionData.fetchOffset);
        } else {
            abortedTransactions = null;
        }
        responseData.put(topicPartition, new PartitionData<>(Errors.NONE, highWatermark, lso, // TODO: should it be changed to the logStartOffset?
        highWatermark, abortedTransactions, kafkaRecords));
        bytesReadable.getAndAdd(kafkaRecords.sizeInBytes());
        tryComplete();
    }, requestHandler.getDecodeExecutor());
}

Example 2 with GetResult

use of org.apache.pulsar.metadata.api.GetResult in project pulsar by apache.

the class RocksdbMetadataStoreTest method testOpenDbWithConfigFile.

@Test
public void testOpenDbWithConfigFile() throws Exception {
    MetadataStore store;
    Path tempDir;
    tempDir = Files.createTempDirectory("RocksdbMetadataStoreTest");
    log.info("Temp dir:{}", tempDir.toAbsolutePath());
    String optionFilePath = getClass().getClassLoader().getResource("rocksdb_option_file_example.ini").getPath();
    log.info("optionFilePath={}", optionFilePath);
    store = MetadataStoreFactory.create("rocksdb:" + tempDir.toAbsolutePath(), MetadataStoreConfig.builder().configFilePath(optionFilePath).build());
    Assert.assertTrue(store instanceof RocksdbMetadataStore);
    String path = "/test";
    byte[] data = "data".getBytes();
    // test put
    CompletableFuture<Stat> f = store.put(path, data, Optional.of(-1L));
    CompletableFuture<Stat> failedPut = store.put(path, data, Optional.of(100L));
    Assert.expectThrows(MetadataStoreException.BadVersionException.class, () -> {
        try {
            failedPut.get();
        } catch (ExecutionException t) {
            throw t.getCause();
        }
    });
    Assert.assertNotNull(f.get());
    log.info("put result:{}", f.get());
    Assert.assertNotNull(store.put(path + "/a", data, Optional.of(-1L)));
    Assert.assertNotNull(store.put(path + "/b", data, Optional.of(-1L)));
    Assert.assertNotNull(store.put(path + "/c", data, Optional.of(-1L)));
    // reopen db
    store.close();
    store = MetadataStoreFactory.create("rocksdb:" + tempDir.toAbsolutePath(), MetadataStoreConfig.builder().configFilePath(optionFilePath).build());
    // test get
    CompletableFuture<Optional<GetResult>> readResult = store.get(path);
    Assert.assertNotNull(readResult.get());
    Assert.assertTrue(readResult.get().isPresent());
    GetResult r = readResult.get().get();
    Assert.assertEquals(path, r.getStat().getPath());
    Assert.assertEquals(0, r.getStat().getVersion());
    Assert.assertEquals(data, r.getValue());
    store.close();
    FileUtils.deleteQuietly(tempDir.toFile());
}

Example 3 with GetResult

use of org.apache.pulsar.metadata.api.GetResult in project pulsar by apache.

the class MetadataStoreBatchingTest method testBigBatchSize.

@Test(dataProvider = "impl")
public void testBigBatchSize(String provider, Supplier<String> urlSupplier) throws Exception {
    @Cleanup MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().batchingEnabled(true).batchingMaxDelayMillis(1_000).build());
    String key1 = newKey();
    // Create 20 MB of data and try to read it out
    int dataSize = 500 * 1024;
    byte[] payload = new byte[dataSize];
    int N = 40;
    List<CompletableFuture<Stat>> putFutures = new ArrayList<>();
    for (int i = 0; i < N; i++) {
        putFutures.add(store.put(key1 + "/" + i, payload, Optional.empty()));
    }
    FutureUtil.waitForAll(putFutures).join();
    List<CompletableFuture<Optional<GetResult>>> getFutures = new ArrayList<>();
    for (int i = 0; i < N; i++) {
        getFutures.add(store.get(key1 + "/" + i));
    }
    FutureUtil.waitForAll(getFutures).join();
}

Example 4 with GetResult

use of org.apache.pulsar.metadata.api.GetResult in project pulsar by apache.

the class MetadataStoreBatchingTest method testBatching.

@Test(dataProvider = "impl")
public void testBatching(String provider, Supplier<String> urlSupplier) throws Exception {
    @Cleanup MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().batchingEnabled(true).batchingMaxDelayMillis(1_000).build());
    String key1 = newKey();
    store.put(key1, new byte[0], Optional.empty()).join();
    String key2 = newKey();
    CompletableFuture<Optional<GetResult>> f1 = store.get(key1);
    CompletableFuture<Optional<GetResult>> f2 = store.get(key2);
    Optional<GetResult> r1 = f1.join();
    Optional<GetResult> r2 = f2.join();
    assertTrue(r1.isPresent());
    assertFalse(r2.isPresent());
}

Example 5 with GetResult

use of org.apache.pulsar.metadata.api.GetResult in project pulsar by apache.

the class MetadataStoreTest method testConcurrentPutGetOneKey.

@Test(dataProvider = "impl")
public void testConcurrentPutGetOneKey(String provider, Supplier<String> urlSupplier) throws Exception {
    MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().build());
    byte[] data = new byte[] { 0 };
    String path = newKey();
    int maxValue = 100;
    store.put(path, data, Optional.of(-1L)).join();
    AtomicInteger successWrites = new AtomicInteger(0);
    Runnable task = new Runnable() {

        @SneakyThrows
        @Override
        public void run() {
            byte value;
            while (true) {
                GetResult readResult = store.get(path).get().get();
                value = (byte) (readResult.getValue()[0] + 1);
                if (value <= maxValue) {
                    CompletableFuture<Void> putResult = store.put(path, new byte[] { value }, Optional.of(readResult.getStat().getVersion())).thenRun(successWrites::incrementAndGet);
                    try {
                        putResult.get();
                    } catch (Exception ignore) {
                    }
                    log.info("Put value {} success:{}. ", value, !putResult.isCompletedExceptionally());
                } else {
                    break;
                }
            }
        }
    };
    CompletableFuture<Void> t1 = CompletableFuture.completedFuture(null).thenRunAsync(task);
    CompletableFuture<Void> t2 = CompletableFuture.completedFuture(null).thenRunAsync(task);
    task.run();
    t1.join();
    t2.join();
    assertFalse(t1.isCompletedExceptionally());
    assertFalse(t2.isCompletedExceptionally());
    assertEquals(successWrites.get(), maxValue);
    assertEquals(store.get(path).get().get().getValue()[0], maxValue);
}