Search in sources :

Example 1 with StreamCreator

use of com.rabbitmq.stream.StreamCreator in project rabbitmq-stream-java-client by rabbitmq.

the class StreamEnvironmentTest method instanciationShouldSucceedWhenLazyInitIsEnabledAndHostIsNotKnown.

@Test
void instanciationShouldSucceedWhenLazyInitIsEnabledAndHostIsNotKnown() {
    String dummyHost = UUID.randomUUID().toString();
    Address dummyAddress = new Address(dummyHost, Client.DEFAULT_PORT);
    try (Environment env = environmentBuilder.host(dummyHost).addressResolver(a -> dummyAddress).lazyInitialization(true).build()) {
        StreamCreator streamCreator = env.streamCreator().stream("should not have been created");
        assertThatThrownBy(() -> streamCreator.create()).isInstanceOf(StreamException.class);
        assertThatThrownBy(() -> env.deleteStream("should not exist")).isInstanceOf(StreamException.class);
        ProducerBuilder producerBuilder = env.producerBuilder().stream(this.stream);
        assertThatThrownBy(() -> producerBuilder.build()).isInstanceOf(StreamException.class);
        ConsumerBuilder consumerBuilder = env.consumerBuilder().stream(this.stream).messageHandler((context, message) -> {
        });
        assertThatThrownBy(() -> consumerBuilder.build()).isInstanceOf(StreamException.class);
    }
}
Also used : ProducerBuilder(com.rabbitmq.stream.ProducerBuilder) Address(com.rabbitmq.stream.Address) Environment(com.rabbitmq.stream.Environment) ConsumerBuilder(com.rabbitmq.stream.ConsumerBuilder) StreamCreator(com.rabbitmq.stream.StreamCreator) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 2 with StreamCreator

use of com.rabbitmq.stream.StreamCreator in project rabbitmq-stream-java-client by rabbitmq.

the class StreamPerfTest method call.

@Override
public Integer call() throws Exception {
    maybeDisplayVersion();
    maybeDisplayEnvironmentVariablesHelp();
    overridePropertiesWithEnvironmentVariables();
    Codec codec = createCodec(this.codecClass);
    ByteBufAllocator byteBufAllocator = ByteBufAllocator.DEFAULT;
    CompositeMeterRegistry meterRegistry = new CompositeMeterRegistry();
    String metricsPrefix = "rabbitmq.stream";
    this.metricsCollector = new MicrometerMetricsCollector(meterRegistry, metricsPrefix);
    Counter producerConfirm = meterRegistry.counter(metricsPrefix + ".producer_confirmed");
    Supplier<String> memoryReportSupplier;
    if (this.memoryReport) {
        long physicalMemory = Utils.physicalMemory();
        String physicalMemoryReport = physicalMemory == 0 ? "" : format(", physical memory %s (%d bytes)", Utils.formatByte(physicalMemory), physicalMemory);
        this.out.println(format("Max memory %s (%d bytes), max direct memory %s (%d bytes)%s", Utils.formatByte(Runtime.getRuntime().maxMemory()), Runtime.getRuntime().maxMemory(), Utils.formatByte(PlatformDependent.maxDirectMemory()), PlatformDependent.maxDirectMemory(), physicalMemoryReport));
        if (byteBufAllocator instanceof ByteBufAllocatorMetricProvider) {
            ByteBufAllocatorMetric allocatorMetric = ((ByteBufAllocatorMetricProvider) byteBufAllocator).metric();
            memoryReportSupplier = () -> {
                long usedHeapMemory = allocatorMetric.usedHeapMemory();
                long usedDirectMemory = allocatorMetric.usedDirectMemory();
                return format("Used heap memory %s (%d bytes), used direct memory %s (%d bytes)", Utils.formatByte(usedHeapMemory), usedHeapMemory, Utils.formatByte(usedDirectMemory), usedDirectMemory);
            };
        } else {
            memoryReportSupplier = () -> "";
        }
    } else {
        memoryReportSupplier = () -> "";
    }
    this.performanceMetrics = new DefaultPerformanceMetrics(meterRegistry, metricsPrefix, this.summaryFile, this.includeByteRates, this.confirmLatency, memoryReportSupplier, this.out);
    // we need to store a long in it
    this.messageSize = this.messageSize < 8 ? 8 : this.messageSize;
    ShutdownService shutdownService = new ShutdownService();
    try {
        MonitoringContext monitoringContext = new MonitoringContext(this.monitoringPort, meterRegistry);
        this.monitorings.forEach(m -> m.configure(monitoringContext));
        monitoringContext.start();
        shutdownService.wrap(closeStep("Closing monitoring context", monitoringContext::close));
        Runtime.getRuntime().addShutdownHook(new Thread(() -> shutdownService.close()));
        // FIXME add confirm latency
        ScheduledExecutorService envExecutor = Executors.newScheduledThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), this.producers), new NamedThreadFactory("stream-perf-test-env-"));
        shutdownService.wrap(closeStep("Closing environment executor", () -> envExecutor.shutdownNow()));
        boolean tls = isTls(this.uris);
        AddressResolver addrResolver;
        if (loadBalancer) {
            int defaultPort = tls ? Client.DEFAULT_TLS_PORT : Client.DEFAULT_PORT;
            List<Address> addresses = this.uris.stream().map(uri -> {
                try {
                    return new URI(uri);
                } catch (URISyntaxException e) {
                    throw new IllegalArgumentException("Error while parsing URI " + uri + ": " + e.getMessage());
                }
            }).map(uriItem -> new Address(uriItem.getHost() == null ? "localhost" : uriItem.getHost(), uriItem.getPort() == -1 ? defaultPort : uriItem.getPort())).collect(Collectors.toList());
            AtomicInteger connectionAttemptCount = new AtomicInteger(0);
            addrResolver = address -> addresses.get(connectionAttemptCount.getAndIncrement() % addresses.size());
        } else {
            if (this.addressResolver == null) {
                addrResolver = address -> address;
            } else {
                // should happen only in tests
                addrResolver = this.addressResolver;
            }
        }
        EnvironmentBuilder environmentBuilder = Environment.builder().id("stream-perf-test").uris(this.uris).addressResolver(addrResolver).scheduledExecutorService(envExecutor).metricsCollector(metricsCollector).byteBufAllocator(byteBufAllocator).codec(codec).maxProducersByConnection(this.producersByConnection).maxTrackingConsumersByConnection(this.trackingConsumersByConnection).maxConsumersByConnection(this.consumersByConnection).rpcTimeout(Duration.ofSeconds(this.rpcTimeout));
        ChannelCustomizer channelCustomizer = channel -> {
        };
        if (tls) {
            TlsConfiguration tlsConfiguration = environmentBuilder.tls();
            tlsConfiguration = tlsConfiguration.sslContext(SslContextBuilder.forClient().trustManager(Utils.TRUST_EVERYTHING_TRUST_MANAGER).build());
            environmentBuilder = tlsConfiguration.environmentBuilder();
            if (!this.sniServerNames.isEmpty()) {
                channelCustomizer = channelCustomizer.andThen(ch -> {
                    SslHandler sslHandler = ch.pipeline().get(SslHandler.class);
                    if (sslHandler != null) {
                        SSLParameters sslParameters = sslHandler.engine().getSSLParameters();
                        sslParameters.setServerNames(this.sniServerNames);
                        sslHandler.engine().setSSLParameters(sslParameters);
                    }
                });
            }
        }
        Environment environment = environmentBuilder.channelCustomizer(channelCustomizer).build();
        shutdownService.wrap(closeStep("Closing environment(s)", () -> environment.close()));
        streams = Utils.streams(this.streamCount, this.streams);
        for (String stream : streams) {
            StreamCreator streamCreator = environment.streamCreator().stream(stream).maxLengthBytes(this.maxLengthBytes).maxSegmentSizeBytes(this.maxSegmentSize).leaderLocator(this.leaderLocator);
            if (this.maxAge != null) {
                streamCreator.maxAge(this.maxAge);
            }
            try {
                streamCreator.create();
            } catch (StreamException e) {
                if (e.getCode() == Constants.RESPONSE_CODE_PRECONDITION_FAILED) {
                    String message = String.format("Warning: stream '%s' already exists, but with different properties than " + "max-length-bytes=%s, stream-max-segment-size-bytes=%s, queue-leader-locator=%s", stream, this.maxLengthBytes, this.maxSegmentSize, this.leaderLocator);
                    if (this.maxAge != null) {
                        message += String.format(", max-age=%s", this.maxAge);
                    }
                    this.out.println(message);
                } else {
                    throw e;
                }
            }
        }
        if (this.deleteStreams) {
            shutdownService.wrap(closeStep("Deleting stream(s)", () -> {
                for (String stream : streams) {
                    LOGGER.debug("Deleting {}", stream);
                    try {
                        environment.deleteStream(stream);
                        LOGGER.debug("Deleted {}", stream);
                    } catch (Exception e) {
                        LOGGER.warn("Could not delete stream {}: {}", stream, e.getMessage());
                    }
                }
            }));
        }
        List<Producer> producers = Collections.synchronizedList(new ArrayList<>(this.producers));
        List<Runnable> producerRunnables = IntStream.range(0, this.producers).mapToObj(i -> {
            Runnable rateLimiterCallback;
            if (this.rate > 0) {
                RateLimiter rateLimiter = RateLimiter.create(this.rate);
                rateLimiterCallback = () -> rateLimiter.acquire(1);
            } else {
                rateLimiterCallback = () -> {
                };
            }
            String stream = stream(this.streams, i);
            ProducerBuilder producerBuilder = environment.producerBuilder();
            String producerName = this.producerNameStrategy.apply(stream, i + 1);
            if (producerName != null && !producerName.trim().isEmpty()) {
                producerBuilder = producerBuilder.name(producerName).confirmTimeout(Duration.ZERO);
            }
            Producer producer = producerBuilder.subEntrySize(this.subEntrySize).batchSize(this.batchSize).compression(this.compression == Compression.NONE ? null : this.compression).maxUnconfirmedMessages(this.confirms).stream(stream).build();
            AtomicLong messageCount = new AtomicLong(0);
            ConfirmationHandler confirmationHandler;
            if (this.confirmLatency) {
                final PerformanceMetrics metrics = this.performanceMetrics;
                final int divisor = Utils.downSamplingDivisor(this.rate);
                confirmationHandler = confirmationStatus -> {
                    if (confirmationStatus.isConfirmed()) {
                        producerConfirm.increment();
                        // this should not affect the metric much
                        if (messageCount.incrementAndGet() % divisor == 0) {
                            try {
                                long time = Utils.readLong(confirmationStatus.getMessage().getBodyAsBinary());
                                // see below why we use current time to measure latency
                                metrics.confirmLatency(System.currentTimeMillis() - time, TimeUnit.MILLISECONDS);
                            } catch (Exception e) {
                            // not able to read the body, something wrong?
                            }
                        }
                    }
                };
            } else {
                confirmationHandler = confirmationStatus -> {
                    if (confirmationStatus.isConfirmed()) {
                        producerConfirm.increment();
                    }
                };
            }
            producers.add(producer);
            return (Runnable) () -> {
                final int msgSize = this.messageSize;
                try {
                    while (true && !Thread.currentThread().isInterrupted()) {
                        rateLimiterCallback.run();
                        // Using current time for interoperability with other tools
                        // and also across different processes.
                        // This is good enough to measure duration/latency this way
                        // in a performance tool.
                        long creationTime = System.currentTimeMillis();
                        byte[] payload = new byte[msgSize];
                        Utils.writeLong(payload, creationTime);
                        producer.send(producer.messageBuilder().addData(payload).build(), confirmationHandler);
                    }
                } catch (Exception e) {
                    if (e instanceof InterruptedException || (e.getCause() != null && e.getCause() instanceof InterruptedException)) {
                        LOGGER.info("Publisher #{} thread interrupted", i, e);
                    } else {
                        LOGGER.warn("Publisher #{} crashed", i, e);
                    }
                }
            };
        }).collect(Collectors.toList());
        List<Consumer> consumers = Collections.synchronizedList(IntStream.range(0, this.consumers).mapToObj(i -> {
            final PerformanceMetrics metrics = this.performanceMetrics;
            AtomicLong messageCount = new AtomicLong(0);
            String stream = stream(streams, i);
            ConsumerBuilder consumerBuilder = environment.consumerBuilder();
            consumerBuilder = consumerBuilder.stream(stream).offset(this.offset);
            if (this.storeEvery > 0) {
                String consumerName = this.consumerNameStrategy.apply(stream, i + 1);
                consumerBuilder = consumerBuilder.name(consumerName).autoTrackingStrategy().messageCountBeforeStorage(this.storeEvery).builder();
            }
            // we assume the publishing rate is the same order as the consuming rate
            // we actually don't want to downsample for low rates
            final int divisor = Utils.downSamplingDivisor(this.rate);
            consumerBuilder = consumerBuilder.messageHandler((context, message) -> {
                // this should not affect the metric much
                if (messageCount.incrementAndGet() % 100 == divisor) {
                    try {
                        long time = Utils.readLong(message.getBodyAsBinary());
                        // see above why we use current time to measure latency
                        metrics.latency(System.currentTimeMillis() - time, TimeUnit.MILLISECONDS);
                    } catch (Exception e) {
                    // not able to read the body, maybe not a message from the
                    // tool
                    }
                    metrics.offset(context.offset());
                }
            });
            Consumer consumer = consumerBuilder.build();
            return consumer;
        }).collect(Collectors.toList()));
        shutdownService.wrap(closeStep("Closing consumers", () -> {
            for (Consumer consumer : consumers) {
                consumer.close();
            }
        }));
        ExecutorService executorService;
        if (this.producers > 0) {
            executorService = Executors.newFixedThreadPool(this.producers, new NamedThreadFactory("stream-perf-test-publishers-"));
            for (Runnable producer : producerRunnables) {
                this.out.println("Starting producer");
                executorService.submit(producer);
            }
        } else {
            executorService = null;
        }
        shutdownService.wrap(closeStep("Closing producers", () -> {
            for (Producer p : producers) {
                p.close();
            }
        }));
        shutdownService.wrap(closeStep("Closing producers executor service", () -> {
            if (executorService != null) {
                executorService.shutdownNow();
            }
        }));
        String metricsHeader = "Arguments: " + String.join(" ", arguments);
        this.performanceMetrics.start(metricsHeader);
        shutdownService.wrap(closeStep("Closing metrics", () -> this.performanceMetrics.close()));
        CountDownLatch latch = new CountDownLatch(1);
        Runtime.getRuntime().addShutdownHook(new Thread(() -> latch.countDown()));
        try {
            latch.await();
        } catch (InterruptedException e) {
        // moving on to the closing sequence
        }
    } finally {
        shutdownService.close();
    }
    return 0;
}
Also used : Arrays(java.util.Arrays) ByteBufAllocatorMetricProvider(io.netty.buffer.ByteBufAllocatorMetricProvider) SNIServerName(javax.net.ssl.SNIServerName) AddressResolver(com.rabbitmq.stream.AddressResolver) ByteBufAllocator(io.netty.buffer.ByteBufAllocator) URISyntaxException(java.net.URISyntaxException) BiFunction(java.util.function.BiFunction) LoggerFactory(org.slf4j.LoggerFactory) Codec(com.rabbitmq.stream.Codec) QpidProtonCodec(com.rabbitmq.stream.codec.QpidProtonCodec) SimpleCodec(com.rabbitmq.stream.codec.SimpleCodec) StreamException(com.rabbitmq.stream.StreamException) PlatformDependent(io.netty.util.internal.PlatformDependent) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) ConsumerBuilder(com.rabbitmq.stream.ConsumerBuilder) CloseCallback(com.rabbitmq.stream.perf.ShutdownService.CloseCallback) Duration(java.time.Duration) Map(java.util.Map) URI(java.net.URI) CommandLine(picocli.CommandLine) Counter(io.micrometer.core.instrument.Counter) PrintWriter(java.io.PrintWriter) ENVIRONMENT_VARIABLE_PREFIX(com.rabbitmq.stream.perf.Utils.ENVIRONMENT_VARIABLE_PREFIX) Collection(java.util.Collection) OPTION_TO_ENVIRONMENT_VARIABLE(com.rabbitmq.stream.perf.Utils.OPTION_TO_ENVIRONMENT_VARIABLE) Compression(com.rabbitmq.stream.compression.Compression) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) Executors(java.util.concurrent.Executors) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) StreamCreator(com.rabbitmq.stream.StreamCreator) SslHandler(io.netty.handler.ssl.SslHandler) ProducerBuilder(com.rabbitmq.stream.ProducerBuilder) Constants(com.rabbitmq.stream.Constants) OffsetSpecification(com.rabbitmq.stream.OffsetSpecification) LeaderLocator(com.rabbitmq.stream.StreamCreator.LeaderLocator) ENVIRONMENT_VARIABLE_LOOKUP(com.rabbitmq.stream.perf.Utils.ENVIRONMENT_VARIABLE_LOOKUP) IntStream(java.util.stream.IntStream) ByteCapacity(com.rabbitmq.stream.ByteCapacity) NamedThreadFactory(com.rabbitmq.stream.perf.Utils.NamedThreadFactory) CompositeMeterRegistry(io.micrometer.core.instrument.composite.CompositeMeterRegistry) SSLParameters(javax.net.ssl.SSLParameters) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) Address(com.rabbitmq.stream.Address) RateLimiter(com.google.common.util.concurrent.RateLimiter) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) ConfirmationHandler(com.rabbitmq.stream.ConfirmationHandler) Charset(java.nio.charset.Charset) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ChannelCustomizer(com.rabbitmq.stream.ChannelCustomizer) ExecutorService(java.util.concurrent.ExecutorService) PrintStream(java.io.PrintStream) Logger(org.slf4j.Logger) MetricsCollector(com.rabbitmq.stream.metrics.MetricsCollector) Environment(com.rabbitmq.stream.Environment) IOException(java.io.IOException) Consumer(com.rabbitmq.stream.Consumer) Producer(com.rabbitmq.stream.Producer) EnvironmentBuilder(com.rabbitmq.stream.EnvironmentBuilder) MicrometerMetricsCollector(com.rabbitmq.stream.metrics.MicrometerMetricsCollector) ByteBufAllocatorMetric(io.netty.buffer.ByteBufAllocatorMetric) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) Client(com.rabbitmq.stream.impl.Client) SslContextBuilder(io.netty.handler.ssl.SslContextBuilder) CommandSpec(picocli.CommandLine.Model.CommandSpec) TlsConfiguration(com.rabbitmq.stream.EnvironmentBuilder.TlsConfiguration) Collections(java.util.Collections) ByteBufAllocator(io.netty.buffer.ByteBufAllocator) Address(com.rabbitmq.stream.Address) MicrometerMetricsCollector(com.rabbitmq.stream.metrics.MicrometerMetricsCollector) ByteBufAllocatorMetric(io.netty.buffer.ByteBufAllocatorMetric) EnvironmentBuilder(com.rabbitmq.stream.EnvironmentBuilder) Codec(com.rabbitmq.stream.Codec) QpidProtonCodec(com.rabbitmq.stream.codec.QpidProtonCodec) SimpleCodec(com.rabbitmq.stream.codec.SimpleCodec) Counter(io.micrometer.core.instrument.Counter) Consumer(com.rabbitmq.stream.Consumer) ConsumerBuilder(com.rabbitmq.stream.ConsumerBuilder) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) NamedThreadFactory(com.rabbitmq.stream.perf.Utils.NamedThreadFactory) SslHandler(io.netty.handler.ssl.SslHandler) RateLimiter(com.google.common.util.concurrent.RateLimiter) ProducerBuilder(com.rabbitmq.stream.ProducerBuilder) Producer(com.rabbitmq.stream.Producer) ConfirmationHandler(com.rabbitmq.stream.ConfirmationHandler) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Environment(com.rabbitmq.stream.Environment) TlsConfiguration(com.rabbitmq.stream.EnvironmentBuilder.TlsConfiguration) ChannelCustomizer(com.rabbitmq.stream.ChannelCustomizer) URISyntaxException(java.net.URISyntaxException) StreamCreator(com.rabbitmq.stream.StreamCreator) URI(java.net.URI) StreamException(com.rabbitmq.stream.StreamException) ByteBufAllocatorMetricProvider(io.netty.buffer.ByteBufAllocatorMetricProvider) SSLParameters(javax.net.ssl.SSLParameters) CompositeMeterRegistry(io.micrometer.core.instrument.composite.CompositeMeterRegistry) AddressResolver(com.rabbitmq.stream.AddressResolver) CountDownLatch(java.util.concurrent.CountDownLatch) URISyntaxException(java.net.URISyntaxException) StreamException(com.rabbitmq.stream.StreamException) IOException(java.io.IOException) AtomicLong(java.util.concurrent.atomic.AtomicLong) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExecutorService(java.util.concurrent.ExecutorService)

Aggregations

Address (com.rabbitmq.stream.Address)2 ConsumerBuilder (com.rabbitmq.stream.ConsumerBuilder)2 Environment (com.rabbitmq.stream.Environment)2 ProducerBuilder (com.rabbitmq.stream.ProducerBuilder)2 StreamCreator (com.rabbitmq.stream.StreamCreator)2 RateLimiter (com.google.common.util.concurrent.RateLimiter)1 AddressResolver (com.rabbitmq.stream.AddressResolver)1 ByteCapacity (com.rabbitmq.stream.ByteCapacity)1 ChannelCustomizer (com.rabbitmq.stream.ChannelCustomizer)1 Codec (com.rabbitmq.stream.Codec)1 ConfirmationHandler (com.rabbitmq.stream.ConfirmationHandler)1 Constants (com.rabbitmq.stream.Constants)1 Consumer (com.rabbitmq.stream.Consumer)1 EnvironmentBuilder (com.rabbitmq.stream.EnvironmentBuilder)1 TlsConfiguration (com.rabbitmq.stream.EnvironmentBuilder.TlsConfiguration)1 OffsetSpecification (com.rabbitmq.stream.OffsetSpecification)1 Producer (com.rabbitmq.stream.Producer)1 LeaderLocator (com.rabbitmq.stream.StreamCreator.LeaderLocator)1 StreamException (com.rabbitmq.stream.StreamException)1 QpidProtonCodec (com.rabbitmq.stream.codec.QpidProtonCodec)1