Examples with Compression com.rabbitmq.stream.compression.Compression used on opensource projects

Example 1 with Compression

use of com.rabbitmq.stream.compression.Compression in project rabbitmq-stream-java-client by rabbitmq.

the class StreamProducerTest method subEntryBatchesSentCompressedShouldBeConsumedProperly.

@Test
void subEntryBatchesSentCompressedShouldBeConsumedProperly() {
    int messagePerProducer = 10000;
    int messageCount = Compression.values().length * messagePerProducer;
    CountDownLatch publishLatch = new CountDownLatch(messageCount);
    ConfirmationHandler confirmationHandler = confirmationStatus -> publishLatch.countDown();
    AtomicInteger messageIndex = new AtomicInteger(0);
    Set<String> publishedBodies = ConcurrentHashMap.newKeySet(messageCount);
    for (Compression compression : Compression.values()) {
        Producer producer = environment.producerBuilder().stream(stream).subEntrySize(100).compression(compression).build();
        IntStream.range(0, messagePerProducer).forEach(i -> {
            String body = "compression " + compression.name() + " message " + messageIndex.getAndIncrement();
            producer.send(producer.messageBuilder().addData(body.getBytes(StandardCharsets.UTF_8)).build(), confirmationHandler);
            publishedBodies.add(body);
        });
    }
    assertThat(latchAssert(publishLatch)).completes();
    Set<String> consumedBodies = ConcurrentHashMap.newKeySet(messageCount);
    CountDownLatch consumeLatch = new CountDownLatch(messageCount);
    environment.consumerBuilder().stream(stream).offset(OffsetSpecification.first()).messageHandler((context, message) -> {
        consumedBodies.add(new String(message.getBodyAsBinary(), StandardCharsets.UTF_8));
        consumeLatch.countDown();
    }).build();
    assertThat(latchAssert(consumeLatch)).completes();
    assertThat(consumedBodies).isNotEmpty().hasSameSizeAs(publishedBodies);
    publishedBodies.forEach(publishBody -> assertThat(consumedBodies.contains(publishBody)).isTrue());
}

Example 2 with Compression

use of com.rabbitmq.stream.compression.Compression in project rabbitmq-stream-java-client by rabbitmq.

the class SubEntryBatchingTest method publishConsumeCompressedMessages.

@ParameterizedTest
@MethodSource("compressionCodecFactories")
void publishConsumeCompressedMessages(CompressionCodecFactory compressionCodecFactory, TestInfo info) {
    Map<Compression, Integer> compressionToReadBytes = new HashMap<>();
    for (Compression compression : Compression.values()) {
        int batchCount = 100;
        int messagesInBatch = 30;
        int messageCount = batchCount * messagesInBatch;
        CountDownLatch publishLatch = new CountDownLatch(batchCount);
        Client publisher = cf.get(new ClientParameters().compressionCodecFactory(compressionCodecFactory).publishConfirmListener((publisherId, publishingId) -> publishLatch.countDown()));
        String s = TestUtils.streamName(info) + "_" + compression.name();
        try {
            Response response = publisher.create(s);
            assertThat(response.isOk()).isTrue();
            response = publisher.declarePublisher(b(0), null, s);
            assertThat(response.isOk()).isTrue();
            Set<String> publishedBodies = ConcurrentHashMap.newKeySet(messageCount);
            IntStream.range(0, batchCount).forEach(batchIndex -> {
                MessageBatch messageBatch = new MessageBatch(compression);
                IntStream.range(0, messagesInBatch).forEach(messageIndex -> {
                    String body = "batch " + batchIndex + " message " + messageIndex;
                    messageBatch.add(publisher.messageBuilder().addData(body.getBytes(UTF8)).build());
                    publishedBodies.add(body);
                });
                publisher.publishBatches(b(0), Collections.singletonList(messageBatch));
            });
            assertThat(latchAssert(publishLatch)).completes();
            Set<String> consumedBodies = ConcurrentHashMap.newKeySet(batchCount * messagesInBatch);
            CountDownLatch consumeLatch = new CountDownLatch(batchCount * messagesInBatch);
            CountMetricsCollector metricsCollector = new CountMetricsCollector();
            Client consumer = cf.get(new ClientParameters().compressionCodecFactory(compressionCodecFactory).chunkListener((client, subscriptionId, offset, messageCount1, dataSize) -> client.credit(subscriptionId, 1)).messageListener((subscriptionId, offset, chunkTimestamp, message) -> {
                consumedBodies.add(new String(message.getBodyAsBinary(), UTF8));
                consumeLatch.countDown();
            }).metricsCollector(metricsCollector));
            response = consumer.subscribe(b(1), s, OffsetSpecification.first(), 2);
            assertThat(response.isOk()).isTrue();
            assertThat(latchAssert(consumeLatch)).completes();
            assertThat(consumedBodies).hasSize(messageCount).hasSameSizeAs(publishedBodies);
            publishedBodies.forEach(publishedBody -> assertThat(consumedBodies.contains(publishedBody)).isTrue());
            compressionToReadBytes.put(compression, metricsCollector.readBytes.get());
        } finally {
            Response response = publisher.delete(s);
            assertThat(response.isOk()).isTrue();
        }
    }
    int plainReadBytes = compressionToReadBytes.get(Compression.NONE);
    Arrays.stream(Compression.values()).filter(comp -> comp != Compression.NONE).forEach(compression -> {
        assertThat(compressionToReadBytes.get(compression)).isLessThan(plainReadBytes);
    });
}

Example 3 with Compression

use of com.rabbitmq.stream.compression.Compression in project rabbitmq-stream-java-client by rabbitmq.

the class StreamPerfTest method call.

@Override
public Integer call() throws Exception {
    maybeDisplayVersion();
    maybeDisplayEnvironmentVariablesHelp();
    overridePropertiesWithEnvironmentVariables();
    Codec codec = createCodec(this.codecClass);
    ByteBufAllocator byteBufAllocator = ByteBufAllocator.DEFAULT;
    CompositeMeterRegistry meterRegistry = new CompositeMeterRegistry();
    String metricsPrefix = "rabbitmq.stream";
    this.metricsCollector = new MicrometerMetricsCollector(meterRegistry, metricsPrefix);
    Counter producerConfirm = meterRegistry.counter(metricsPrefix + ".producer_confirmed");
    Supplier<String> memoryReportSupplier;
    if (this.memoryReport) {
        long physicalMemory = Utils.physicalMemory();
        String physicalMemoryReport = physicalMemory == 0 ? "" : format(", physical memory %s (%d bytes)", Utils.formatByte(physicalMemory), physicalMemory);
        this.out.println(format("Max memory %s (%d bytes), max direct memory %s (%d bytes)%s", Utils.formatByte(Runtime.getRuntime().maxMemory()), Runtime.getRuntime().maxMemory(), Utils.formatByte(PlatformDependent.maxDirectMemory()), PlatformDependent.maxDirectMemory(), physicalMemoryReport));
        if (byteBufAllocator instanceof ByteBufAllocatorMetricProvider) {
            ByteBufAllocatorMetric allocatorMetric = ((ByteBufAllocatorMetricProvider) byteBufAllocator).metric();
            memoryReportSupplier = () -> {
                long usedHeapMemory = allocatorMetric.usedHeapMemory();
                long usedDirectMemory = allocatorMetric.usedDirectMemory();
                return format("Used heap memory %s (%d bytes), used direct memory %s (%d bytes)", Utils.formatByte(usedHeapMemory), usedHeapMemory, Utils.formatByte(usedDirectMemory), usedDirectMemory);
            };
        } else {
            memoryReportSupplier = () -> "";
        }
    } else {
        memoryReportSupplier = () -> "";
    }
    this.performanceMetrics = new DefaultPerformanceMetrics(meterRegistry, metricsPrefix, this.summaryFile, this.includeByteRates, this.confirmLatency, memoryReportSupplier, this.out);
    // we need to store a long in it
    this.messageSize = this.messageSize < 8 ? 8 : this.messageSize;
    ShutdownService shutdownService = new ShutdownService();
    try {
        MonitoringContext monitoringContext = new MonitoringContext(this.monitoringPort, meterRegistry);
        this.monitorings.forEach(m -> m.configure(monitoringContext));
        monitoringContext.start();
        shutdownService.wrap(closeStep("Closing monitoring context", monitoringContext::close));
        Runtime.getRuntime().addShutdownHook(new Thread(() -> shutdownService.close()));
        // FIXME add confirm latency
        ScheduledExecutorService envExecutor = Executors.newScheduledThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), this.producers), new NamedThreadFactory("stream-perf-test-env-"));
        shutdownService.wrap(closeStep("Closing environment executor", () -> envExecutor.shutdownNow()));
        boolean tls = isTls(this.uris);
        AddressResolver addrResolver;
        if (loadBalancer) {
            int defaultPort = tls ? Client.DEFAULT_TLS_PORT : Client.DEFAULT_PORT;
            List<Address> addresses = this.uris.stream().map(uri -> {
                try {
                    return new URI(uri);
                } catch (URISyntaxException e) {
                    throw new IllegalArgumentException("Error while parsing URI " + uri + ": " + e.getMessage());
                }
            }).map(uriItem -> new Address(uriItem.getHost() == null ? "localhost" : uriItem.getHost(), uriItem.getPort() == -1 ? defaultPort : uriItem.getPort())).collect(Collectors.toList());
            AtomicInteger connectionAttemptCount = new AtomicInteger(0);
            addrResolver = address -> addresses.get(connectionAttemptCount.getAndIncrement() % addresses.size());
        } else {
            if (this.addressResolver == null) {
                addrResolver = address -> address;
            } else {
                // should happen only in tests
                addrResolver = this.addressResolver;
            }
        }
        EnvironmentBuilder environmentBuilder = Environment.builder().id("stream-perf-test").uris(this.uris).addressResolver(addrResolver).scheduledExecutorService(envExecutor).metricsCollector(metricsCollector).byteBufAllocator(byteBufAllocator).codec(codec).maxProducersByConnection(this.producersByConnection).maxTrackingConsumersByConnection(this.trackingConsumersByConnection).maxConsumersByConnection(this.consumersByConnection).rpcTimeout(Duration.ofSeconds(this.rpcTimeout));
        ChannelCustomizer channelCustomizer = channel -> {
        };
        if (tls) {
            TlsConfiguration tlsConfiguration = environmentBuilder.tls();
            tlsConfiguration = tlsConfiguration.sslContext(SslContextBuilder.forClient().trustManager(Utils.TRUST_EVERYTHING_TRUST_MANAGER).build());
            environmentBuilder = tlsConfiguration.environmentBuilder();
            if (!this.sniServerNames.isEmpty()) {
                channelCustomizer = channelCustomizer.andThen(ch -> {
                    SslHandler sslHandler = ch.pipeline().get(SslHandler.class);
                    if (sslHandler != null) {
                        SSLParameters sslParameters = sslHandler.engine().getSSLParameters();
                        sslParameters.setServerNames(this.sniServerNames);
                        sslHandler.engine().setSSLParameters(sslParameters);
                    }
                });
            }
        }
        Environment environment = environmentBuilder.channelCustomizer(channelCustomizer).build();
        shutdownService.wrap(closeStep("Closing environment(s)", () -> environment.close()));
        streams = Utils.streams(this.streamCount, this.streams);
        for (String stream : streams) {
            StreamCreator streamCreator = environment.streamCreator().stream(stream).maxLengthBytes(this.maxLengthBytes).maxSegmentSizeBytes(this.maxSegmentSize).leaderLocator(this.leaderLocator);
            if (this.maxAge != null) {
                streamCreator.maxAge(this.maxAge);
            }
            try {
                streamCreator.create();
            } catch (StreamException e) {
                if (e.getCode() == Constants.RESPONSE_CODE_PRECONDITION_FAILED) {
                    String message = String.format("Warning: stream '%s' already exists, but with different properties than " + "max-length-bytes=%s, stream-max-segment-size-bytes=%s, queue-leader-locator=%s", stream, this.maxLengthBytes, this.maxSegmentSize, this.leaderLocator);
                    if (this.maxAge != null) {
                        message += String.format(", max-age=%s", this.maxAge);
                    }
                    this.out.println(message);
                } else {
                    throw e;
                }
            }
        }
        if (this.deleteStreams) {
            shutdownService.wrap(closeStep("Deleting stream(s)", () -> {
                for (String stream : streams) {
                    LOGGER.debug("Deleting {}", stream);
                    try {
                        environment.deleteStream(stream);
                        LOGGER.debug("Deleted {}", stream);
                    } catch (Exception e) {
                        LOGGER.warn("Could not delete stream {}: {}", stream, e.getMessage());
                    }
                }
            }));
        }
        List<Producer> producers = Collections.synchronizedList(new ArrayList<>(this.producers));
        List<Runnable> producerRunnables = IntStream.range(0, this.producers).mapToObj(i -> {
            Runnable rateLimiterCallback;
            if (this.rate > 0) {
                RateLimiter rateLimiter = RateLimiter.create(this.rate);
                rateLimiterCallback = () -> rateLimiter.acquire(1);
            } else {
                rateLimiterCallback = () -> {
                };
            }
            String stream = stream(this.streams, i);
            ProducerBuilder producerBuilder = environment.producerBuilder();
            String producerName = this.producerNameStrategy.apply(stream, i + 1);
            if (producerName != null && !producerName.trim().isEmpty()) {
                producerBuilder = producerBuilder.name(producerName).confirmTimeout(Duration.ZERO);
            }
            Producer producer = producerBuilder.subEntrySize(this.subEntrySize).batchSize(this.batchSize).compression(this.compression == Compression.NONE ? null : this.compression).maxUnconfirmedMessages(this.confirms).stream(stream).build();
            AtomicLong messageCount = new AtomicLong(0);
            ConfirmationHandler confirmationHandler;
            if (this.confirmLatency) {
                final PerformanceMetrics metrics = this.performanceMetrics;
                final int divisor = Utils.downSamplingDivisor(this.rate);
                confirmationHandler = confirmationStatus -> {
                    if (confirmationStatus.isConfirmed()) {
                        producerConfirm.increment();
                        // this should not affect the metric much
                        if (messageCount.incrementAndGet() % divisor == 0) {
                            try {
                                long time = Utils.readLong(confirmationStatus.getMessage().getBodyAsBinary());
                                // see below why we use current time to measure latency
                                metrics.confirmLatency(System.currentTimeMillis() - time, TimeUnit.MILLISECONDS);
                            } catch (Exception e) {
                            // not able to read the body, something wrong?
                            }
                        }
                    }
                };
            } else {
                confirmationHandler = confirmationStatus -> {
                    if (confirmationStatus.isConfirmed()) {
                        producerConfirm.increment();
                    }
                };
            }
            producers.add(producer);
            return (Runnable) () -> {
                final int msgSize = this.messageSize;
                try {
                    while (true && !Thread.currentThread().isInterrupted()) {
                        rateLimiterCallback.run();
                        // Using current time for interoperability with other tools
                        // and also across different processes.
                        // This is good enough to measure duration/latency this way
                        // in a performance tool.
                        long creationTime = System.currentTimeMillis();
                        byte[] payload = new byte[msgSize];
                        Utils.writeLong(payload, creationTime);
                        producer.send(producer.messageBuilder().addData(payload).build(), confirmationHandler);
                    }
                } catch (Exception e) {
                    if (e instanceof InterruptedException || (e.getCause() != null && e.getCause() instanceof InterruptedException)) {
                        LOGGER.info("Publisher #{} thread interrupted", i, e);
                    } else {
                        LOGGER.warn("Publisher #{} crashed", i, e);
                    }
                }
            };
        }).collect(Collectors.toList());
        List<Consumer> consumers = Collections.synchronizedList(IntStream.range(0, this.consumers).mapToObj(i -> {
            final PerformanceMetrics metrics = this.performanceMetrics;
            AtomicLong messageCount = new AtomicLong(0);
            String stream = stream(streams, i);
            ConsumerBuilder consumerBuilder = environment.consumerBuilder();
            consumerBuilder = consumerBuilder.stream(stream).offset(this.offset);
            if (this.storeEvery > 0) {
                String consumerName = this.consumerNameStrategy.apply(stream, i + 1);
                consumerBuilder = consumerBuilder.name(consumerName).autoTrackingStrategy().messageCountBeforeStorage(this.storeEvery).builder();
            }
            // we assume the publishing rate is the same order as the consuming rate
            // we actually don't want to downsample for low rates
            final int divisor = Utils.downSamplingDivisor(this.rate);
            consumerBuilder = consumerBuilder.messageHandler((context, message) -> {
                // this should not affect the metric much
                if (messageCount.incrementAndGet() % 100 == divisor) {
                    try {
                        long time = Utils.readLong(message.getBodyAsBinary());
                        // see above why we use current time to measure latency
                        metrics.latency(System.currentTimeMillis() - time, TimeUnit.MILLISECONDS);
                    } catch (Exception e) {
                    // not able to read the body, maybe not a message from the
                    // tool
                    }
                    metrics.offset(context.offset());
                }
            });
            Consumer consumer = consumerBuilder.build();
            return consumer;
        }).collect(Collectors.toList()));
        shutdownService.wrap(closeStep("Closing consumers", () -> {
            for (Consumer consumer : consumers) {
                consumer.close();
            }
        }));
        ExecutorService executorService;
        if (this.producers > 0) {
            executorService = Executors.newFixedThreadPool(this.producers, new NamedThreadFactory("stream-perf-test-publishers-"));
            for (Runnable producer : producerRunnables) {
                this.out.println("Starting producer");
                executorService.submit(producer);
            }
        } else {
            executorService = null;
        }
        shutdownService.wrap(closeStep("Closing producers", () -> {
            for (Producer p : producers) {
                p.close();
            }
        }));
        shutdownService.wrap(closeStep("Closing producers executor service", () -> {
            if (executorService != null) {
                executorService.shutdownNow();
            }
        }));
        String metricsHeader = "Arguments: " + String.join(" ", arguments);
        this.performanceMetrics.start(metricsHeader);
        shutdownService.wrap(closeStep("Closing metrics", () -> this.performanceMetrics.close()));
        CountDownLatch latch = new CountDownLatch(1);
        Runtime.getRuntime().addShutdownHook(new Thread(() -> latch.countDown()));
        try {
            latch.await();
        } catch (InterruptedException e) {
        // moving on to the closing sequence
        }
    } finally {
        shutdownService.close();
    }
    return 0;
}

Example 4 with Compression

use of com.rabbitmq.stream.compression.Compression in project rabbitmq-stream-java-client by rabbitmq.

the class SubEntryBatchingTest method subEntriesCompressedWithDifferentCompressionsShouldBeReadCorrectly.

@Test
void subEntriesCompressedWithDifferentCompressionsShouldBeReadCorrectly() {
    List<CompressionCodecFactory> compressionCodecFactories = compressionCodecFactories().collect(Collectors.toList());
    int batchCount = compressionCodecFactories.size() * Compression.values().length;
    int messagesInBatch = 30;
    int messageCount = batchCount * messagesInBatch;
    AtomicInteger messageIndex = new AtomicInteger(0);
    CountDownLatch publishLatch = new CountDownLatch(batchCount);
    Set<String> publishedBodies = ConcurrentHashMap.newKeySet(messageCount);
    compressionCodecFactories.forEach(compressionCodecFactory -> {
        Client publisher = cf.get(new ClientParameters().compressionCodecFactory(compressionCodecFactory).publishConfirmListener((publisherId, publishingId) -> publishLatch.countDown()));
        Response response = publisher.declarePublisher(b(0), null, stream);
        assertThat(response.isOk()).isTrue();
        for (Compression compression : Compression.values()) {
            MessageBatch messageBatch = new MessageBatch(compression);
            IntStream.range(0, messagesInBatch).forEach(i -> {
                String body = "compression " + compression.name() + " message " + messageIndex.getAndIncrement();
                messageBatch.add(publisher.messageBuilder().addData(body.getBytes(UTF8)).build());
                publishedBodies.add(body);
            });
            publisher.publishBatches(b(0), Collections.singletonList(messageBatch));
        }
    });
    assertThat(latchAssert(publishLatch)).completes();
    compressionCodecFactories.forEach(compressionCodecFactory -> {
        CountDownLatch consumeLatch = new CountDownLatch(messageCount);
        Set<String> consumedBodies = ConcurrentHashMap.newKeySet(messageCount);
        Client consumer = cf.get(new ClientParameters().compressionCodecFactory(compressionCodecFactory).chunkListener((client, subscriptionId, offset, messageCount1, dataSize) -> client.credit(subscriptionId, 1)).messageListener((subscriptionId, offset, chunkTimestamp, message) -> {
            consumedBodies.add(new String(message.getBodyAsBinary(), UTF8));
            consumeLatch.countDown();
        }));
        Response response = consumer.subscribe(b(1), stream, OffsetSpecification.first(), 2);
        assertThat(response.isOk()).isTrue();
        assertThat(latchAssert(consumeLatch)).completes();
        assertThat(consumedBodies).hasSize(messageCount).hasSameSizeAs(publishedBodies);
        publishedBodies.forEach(publishBody -> assertThat(consumedBodies.contains(publishBody)).isTrue());
    });
}