Examples with DestinationName com.yahoo.pulsar.common.naming.DestinationName used on opensource projects

Example 51 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ServerConnection method sendPartitionMetadataResponse.

private void sendPartitionMetadataResponse(CommandPartitionedTopicMetadata partitionMetadata) {
    final long requestId = partitionMetadata.getRequestId();
    DestinationName dn = DestinationName.get(partitionMetadata.getTopic());
    service.getDiscoveryProvider().getPartitionedTopicMetadata(service, dn, authRole).thenAccept(metadata -> {
        if (LOG.isDebugEnabled()) {
            LOG.debug("[{}] Total number of partitions for topic {} is {}", authRole, dn, metadata.partitions);
        }
        ctx.writeAndFlush(Commands.newPartitionMetadataResponse(metadata.partitions, requestId));
    }).exceptionally(ex -> {
        LOG.warn("[{}] Failed to get partitioned metadata for topic {} {}", remoteAddress, dn, ex.getMessage(), ex);
        ctx.writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.ServiceNotReady, ex.getMessage(), requestId));
        return null;
    });
}

Example 52 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class PerformanceConsumer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-consumer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topic.size() != 1) {
        System.out.println("Only one destination name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
    }
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments));
    final DestinationName prefixDestinationName = DestinationName.get(arguments.topic.get(0));
    final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null;
    MessageListener listener = new MessageListener() {

        public void received(Consumer consumer, Message msg) {
            messagesReceived.increment();
            bytesReceived.add(msg.getData().length);
            if (limiter != null) {
                limiter.acquire();
            }
            consumer.acknowledgeAsync(msg);
        }
    };
    EventLoopGroup eventLoopGroup;
    if (SystemUtils.IS_OS_LINUX) {
        eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2, new DefaultThreadFactory("pulsar-perf-consumer"));
    } else {
        eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-consumer"));
    }
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setConnectionsPerBroker(arguments.maxConnections);
    clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient pulsarClient = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup);
    List<Future<Consumer>> futures = Lists.newArrayList();
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setMessageListener(listener);
    consumerConfig.setReceiverQueueSize(arguments.receiverQueueSize);
    for (int i = 0; i < arguments.numDestinations; i++) {
        final DestinationName destinationName = (arguments.numDestinations == 1) ? prefixDestinationName : DestinationName.get(String.format("%s-%d", prefixDestinationName, i));
        log.info("Adding {} consumers on destination {}", arguments.numConsumers, destinationName);
        for (int j = 0; j < arguments.numConsumers; j++) {
            String subscriberName;
            if (arguments.numConsumers > 1) {
                subscriberName = String.format("%s-%d", arguments.subscriberName, j);
            } else {
                subscriberName = arguments.subscriberName;
            }
            futures.add(pulsarClient.subscribeAsync(destinationName.toString(), subscriberName, consumerConfig));
        }
    }
    for (Future<Consumer> future : futures) {
        future.get();
    }
    log.info("Start receiving from {} consumers on {} destinations", arguments.numConsumers, arguments.numDestinations);
    long oldTime = System.nanoTime();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesReceived.sumThenReset() / elapsed;
        double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024;
        log.info("Throughput received: {}  msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput));
        oldTime = now;
    }
    pulsarClient.close();
}

Example 53 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class AbstractWebSocketHandler method extractTopicName.

private String extractTopicName(HttpServletRequest request) {
    String uri = request.getRequestURI();
    List<String> parts = Splitter.on("/").splitToList(uri);
    // Format must be like :
    // /ws/producer/persistent/my-property/my-cluster/my-ns/my-topic
    // or
    // /ws/consumer/persistent/my-property/my-cluster/my-ns/my-topic/my-subscription
    checkArgument(parts.size() >= 8, "Invalid topic name format");
    checkArgument(parts.get(1).equals("ws"));
    checkArgument(parts.get(3).equals("persistent"));
    DestinationName dn = DestinationName.get("persistent", parts.get(4), parts.get(5), parts.get(6), parts.get(7));
    return dn.toString();
}

Example 54 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class BrokerService method isBacklogExceeded.

/**
     *
     * @param topic
     *            needing quota enforcement check
     * @return determine if quota enforcement needs to be done for topic
     */
public boolean isBacklogExceeded(PersistentTopic topic) {
    DestinationName destination = DestinationName.get(topic.getName());
    long backlogQuotaLimitInBytes = getBacklogQuotaManager().getBacklogQuotaLimit(destination.getNamespace());
    if (log.isDebugEnabled()) {
        log.debug("[{}] - backlog quota limit = [{}]", topic.getName(), backlogQuotaLimitInBytes);
    }
    // check if backlog exceeded quota
    long storageSize = topic.getBacklogSize();
    if (log.isDebugEnabled()) {
        log.debug("[{}] Storage size = [{}], limit [{}]", topic.getName(), storageSize, backlogQuotaLimitInBytes);
    }
    return (storageSize >= backlogQuotaLimitInBytes);
}

Example 55 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class BrokerService method createPersistentTopic.

private CompletableFuture<Topic> createPersistentTopic(final String topic) throws RuntimeException {
    checkTopicNsOwnership(topic);
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    DestinationName destinationName = DestinationName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(destinationName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        throw new RuntimeException(new ServiceUnitNotReadyException(msg));
    }
    final CompletableFuture<Topic> topicFuture = new CompletableFuture<>();
    getManagedLedgerConfig(destinationName).thenAccept(config -> {
        managedLedgerFactory.asyncOpen(destinationName.getPersistenceNamingEncoding(), config, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                PersistentTopic persistentTopic = new PersistentTopic(topic, ledger, BrokerService.this);
                CompletableFuture<Void> replicationFuture = persistentTopic.checkReplication();
                replicationFuture.thenRun(() -> {
                    log.info("Created topic {}", topic);
                    long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                    pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                    addTopicToStatsMaps(destinationName, persistentTopic);
                    topicFuture.complete(persistentTopic);
                });
                replicationFuture.exceptionally((ex) -> {
                    log.warn("Replication check failed. Removing topic from topics list {}, {}", topic, ex);
                    persistentTopic.stopReplProducers().whenComplete((v, exception) -> {
                        topics.remove(topic, topicFuture);
                        topicFuture.completeExceptionally(ex);
                    });
                    return null;
                });
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                log.warn("Failed to create topic {}", topic, exception);
                topics.remove(topic, topicFuture);
                topicFuture.completeExceptionally(new PersistenceException(exception));
            }
        }, null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        topics.remove(topic, topicFuture);
        topicFuture.completeExceptionally(exception);
        return null;
    });
    return topicFuture;
}