Examples with PartitionedTopicMetadata com.yahoo.pulsar.common.partition.PartitionedTopicMetadata used on opensource projects

Example 6 with PartitionedTopicMetadata

use of com.yahoo.pulsar.common.partition.PartitionedTopicMetadata in project pulsar by yahoo.

the class PersistentTopics method peekNthMessage.

@GET
@Path("/{property}/{cluster}/{namespace}/{destination}/subscription/{subName}/position/{messagePosition}")
@ApiOperation(value = "Peek nth message on a topic subscription.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic, subscription or the message position does not exist") })
public Response peekNthMessage(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination, @PathParam("subName") String subName, @PathParam("messagePosition") int messagePosition, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    destination = decode(destination);
    DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(property, cluster, namespace, destination, authoritative);
    if (partitionMetadata.partitions > 0) {
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Peek messages on a partitioned topic is not allowed");
    }
    validateAdminOperationOnDestination(dn, authoritative);
    PersistentTopic topic = getTopicReference(dn);
    PersistentReplicator repl = null;
    PersistentSubscription sub = null;
    Entry entry = null;
    if (subName.startsWith(topic.replicatorPrefix)) {
        repl = getReplicatorReference(subName, topic);
    } else {
        sub = getSubscriptionReference(subName, topic);
    }
    try {
        if (subName.startsWith(topic.replicatorPrefix)) {
            entry = repl.peekNthMessage(messagePosition).get();
        } else {
            entry = sub.peekNthMessage(messagePosition).get();
        }
        checkNotNull(entry);
        PositionImpl pos = (PositionImpl) entry.getPosition();
        ByteBuf metadataAndPayload = entry.getDataBuffer();
        // moves the readerIndex to the payload
        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
        ResponseBuilder responseBuilder = Response.ok();
        responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
        for (KeyValue keyValue : metadata.getPropertiesList()) {
            responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
        }
        if (metadata.hasPublishTime()) {
            responseBuilder.header("X-Pulsar-publish-time", DATE_FORMAT.format(Instant.ofEpochMilli(metadata.getPublishTime())));
        }
        // Decode if needed
        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
        ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
        // Copy into a heap buffer for output stream compatibility
        ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(), uncompressedPayload.readableBytes());
        data.writeBytes(uncompressedPayload);
        uncompressedPayload.release();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream output) throws IOException, WebApplicationException {
                output.write(data.array(), data.arrayOffset(), data.readableBytes());
                data.release();
            }
        };
        return responseBuilder.entity(stream).build();
    } catch (NullPointerException npe) {
        throw new RestException(Status.NOT_FOUND, "Message not found");
    } catch (Exception exception) {
        log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, dn, subName, exception);
        throw new RestException(exception);
    } finally {
        if (entry != null) {
            entry.release();
        }
    }
}

Example 7 with PartitionedTopicMetadata

use of com.yahoo.pulsar.common.partition.PartitionedTopicMetadata in project pulsar by yahoo.

the class PersistentTopics method getPartitionedTopicMetadata.

public PartitionedTopicMetadata getPartitionedTopicMetadata(String property, String cluster, String namespace, String destination, boolean authoritative) {
    DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
    validateClusterOwnership(dn.getCluster());
    try {
        checkConnect(dn);
    } catch (WebApplicationException e) {
        validateAdminAccessOnProperty(dn.getProperty());
    } catch (Exception e) {
        // unknown error marked as internal server error
        log.warn("Unexpected error while authorizing lookup. destination={}, role={}. Error: {}", destination, clientAppId(), e.getMessage(), e);
        throw new RestException(e);
    }
    String path = path(PARTITIONED_TOPIC_PATH_ZNODE, property, cluster, namespace, domain(), dn.getEncodedLocalName());
    PartitionedTopicMetadata partitionMetadata = fetchPartitionedTopicMetadata(pulsar(), path);
    if (log.isDebugEnabled()) {
        log.debug("[{}] Total number of partitions for topic {} is {}", clientAppId(), dn, partitionMetadata.partitions);
    }
    return partitionMetadata;
}

Example 8 with PartitionedTopicMetadata

use of com.yahoo.pulsar.common.partition.PartitionedTopicMetadata in project pulsar by yahoo.

the class BinaryProtoLookupService method getPartitionedTopicMetadata.

private CompletableFuture<PartitionedTopicMetadata> getPartitionedTopicMetadata(InetSocketAddress socketAddress, DestinationName destination) {
    CompletableFuture<PartitionedTopicMetadata> partitionFuture = new CompletableFuture<PartitionedTopicMetadata>();
    client.getCnxPool().getConnection(socketAddress).thenAccept(clientCnx -> {
        long requestId = client.newRequestId();
        ByteBuf request = Commands.newPartitionMetadataRequest(destination.toString(), requestId);
        clientCnx.newLookup(request, requestId).thenAccept(lookupDataResult -> {
            try {
                partitionFuture.complete(new PartitionedTopicMetadata(lookupDataResult.partitions));
            } catch (Exception e) {
                partitionFuture.completeExceptionally(new PulsarClientException.LookupException(format("Failed to parse partition-response redirect=%s , partitions with %s", lookupDataResult.redirect, lookupDataResult.partitions, e.getMessage())));
            }
        }).exceptionally((e) -> {
            log.warn("[{}] failed to get Partitioned metadata : {}", destination.toString(), e.getCause().getMessage(), e);
            partitionFuture.completeExceptionally(e);
            return null;
        });
    }).exceptionally(connectionException -> {
        partitionFuture.completeExceptionally(connectionException);
        return null;
    });
    return partitionFuture;
}

Example 9 with PartitionedTopicMetadata

use of com.yahoo.pulsar.common.partition.PartitionedTopicMetadata in project pulsar by yahoo.

the class PersistentTopicsImpl method getPartitionedTopicMetadataAsync.

@Override
public CompletableFuture<PartitionedTopicMetadata> getPartitionedTopicMetadataAsync(String destination) {
    DestinationName ds = validateTopic(destination);
    final CompletableFuture<PartitionedTopicMetadata> future = new CompletableFuture<>();
    asyncGetRequest(persistentTopics.path(ds.getNamespace()).path(ds.getEncodedLocalName()).path("partitions"), new InvocationCallback<PartitionedTopicMetadata>() {

        @Override
        public void completed(PartitionedTopicMetadata response) {
            future.complete(response);
        }

        @Override
        public void failed(Throwable throwable) {
            future.completeExceptionally(getApiException(throwable.getCause()));
        }
    });
    return future;
}

Example 10 with PartitionedTopicMetadata

use of com.yahoo.pulsar.common.partition.PartitionedTopicMetadata in project pulsar by yahoo.

the class AdminApiTest method testPulsarAdminForUriAndUrlEncoding.

/**
     * This test-case verifies that broker should support both url/uri encoding for topic-name. It calls below api with
     * url-encoded and also uri-encoded topic-name in http request: a. PartitionedMetadataLookup b. TopicLookup c. Topic
     * Stats
     * 
     * @param topicName
     * @throws Exception
     */
@Test(dataProvider = "topicName")
public void testPulsarAdminForUriAndUrlEncoding(String topicName) throws Exception {
    final String ns1 = "prop-xyz/use/ns1";
    final String dn1 = "persistent://" + ns1 + "/" + topicName;
    final String urlEncodedTopic = Codec.encode(topicName);
    final String uriEncodedTopic = urlEncodedTopic.replaceAll("\\+", "%20");
    final int numOfPartitions = 4;
    admin.persistentTopics().createPartitionedTopic(dn1, numOfPartitions);
    // Create a consumer to get stats on this topic
    Consumer consumer1 = pulsarClient.subscribe(dn1, "my-subscriber-name", new ConsumerConfiguration());
    PersistentTopicsImpl persistent = (PersistentTopicsImpl) admin.persistentTopics();
    Field field = PersistentTopicsImpl.class.getDeclaredField("persistentTopics");
    field.setAccessible(true);
    WebTarget persistentTopics = (WebTarget) field.get(persistent);
    // (1) Get PartitionedMetadata : with Url and Uri encoding
    final CompletableFuture<PartitionedTopicMetadata> urlEncodedPartitionedMetadata = new CompletableFuture<>();
    // (a) Url encoding
    persistent.asyncGetRequest(persistentTopics.path(ns1).path(urlEncodedTopic).path("partitions"), new InvocationCallback<PartitionedTopicMetadata>() {

        @Override
        public void completed(PartitionedTopicMetadata response) {
            urlEncodedPartitionedMetadata.complete(response);
        }

        @Override
        public void failed(Throwable e) {
            Assert.fail(e.getMessage());
        }
    });
    final CompletableFuture<PartitionedTopicMetadata> uriEncodedPartitionedMetadata = new CompletableFuture<>();
    // (b) Uri encoding
    persistent.asyncGetRequest(persistentTopics.path(ns1).path(uriEncodedTopic).path("partitions"), new InvocationCallback<PartitionedTopicMetadata>() {

        @Override
        public void completed(PartitionedTopicMetadata response) {
            uriEncodedPartitionedMetadata.complete(response);
        }

        @Override
        public void failed(Throwable e) {
            uriEncodedPartitionedMetadata.completeExceptionally(e);
        }
    });
    assertEquals(urlEncodedPartitionedMetadata.get().partitions, numOfPartitions);
    assertEquals(urlEncodedPartitionedMetadata.get().partitions, (uriEncodedPartitionedMetadata.get().partitions));
    // (2) Get Topic Lookup
    LookupImpl lookup = (LookupImpl) admin.lookups();
    Field field2 = LookupImpl.class.getDeclaredField("v2lookup");
    field2.setAccessible(true);
    WebTarget target2 = (WebTarget) field2.get(lookup);
    // (a) Url encoding
    LookupData urlEncodedLookupData = lookup.request(target2.path("/destination/persistent").path(ns1 + "/" + urlEncodedTopic)).get(LookupData.class);
    // (b) Uri encoding
    LookupData uriEncodedLookupData = lookup.request(target2.path("/destination/persistent").path(ns1 + "/" + uriEncodedTopic)).get(LookupData.class);
    Assert.assertNotNull(urlEncodedLookupData.getBrokerUrl());
    assertEquals(urlEncodedLookupData.getBrokerUrl(), uriEncodedLookupData.getBrokerUrl());
    // (3) Get Topic Stats
    final CompletableFuture<PersistentTopicStats> urlStats = new CompletableFuture<>();
    // (a) Url encoding
    persistent.asyncGetRequest(persistentTopics.path(ns1).path(urlEncodedTopic + "-partition-1").path("stats"), new InvocationCallback<PersistentTopicStats>() {

        @Override
        public void completed(PersistentTopicStats response) {
            urlStats.complete(response);
        }

        @Override
        public void failed(Throwable e) {
            urlStats.completeExceptionally(e);
        }
    });
    // (b) Uri encoding
    final CompletableFuture<PersistentTopicStats> uriStats = new CompletableFuture<>();
    persistent.asyncGetRequest(persistentTopics.path(ns1).path(uriEncodedTopic + "-partition-1").path("stats"), new InvocationCallback<PersistentTopicStats>() {

        @Override
        public void completed(PersistentTopicStats response) {
            uriStats.complete(response);
        }

        @Override
        public void failed(Throwable e) {
            uriStats.completeExceptionally(e);
        }
    });
    assertEquals(urlStats.get().subscriptions.size(), 1);
    assertEquals(uriStats.get().subscriptions.size(), 1);
}