Examples with Codec com.rabbitmq.stream.Codec used on opensource projects

Example 1 with Codec

use of com.rabbitmq.stream.Codec in project rabbitmq-stream-java-client by rabbitmq.

the class CodecsTest method codecsCouples.

static Iterable<CodecCouple> codecsCouples() {
    List<Codec> codecs = Arrays.asList(new QpidProtonCodec(), new SwiftMqCodec());
    List<CodecCouple> couples = new ArrayList<>();
    for (Codec serializer : codecs) {
        for (Codec deserializer : codecs) {
            couples.add(new CodecCouple(serializer, deserializer, () -> serializer.messageBuilder()));
            couples.add(new CodecCouple(serializer, deserializer, () -> new WrapperMessageBuilder()));
        }
    }
    return couples;
}

Example 2 with Codec

use of com.rabbitmq.stream.Codec in project rabbitmq-stream-java-client by rabbitmq.

the class CodecsTest method codecs.

@ParameterizedTest
@MethodSource("codecsCouples")
void codecs(CodecCouple codecCouple) {
    Codec serializer = codecCouple.serializer;
    Codec deserializer = codecCouple.deserializer;
    Stream<MessageTestConfiguration> messageOperations = Stream.of(test(builder -> builder.properties().messageId(42).messageBuilder(), message -> assertThat(message.getProperties().getMessageIdAsLong()).isEqualTo(42)), test(builder -> builder.properties().messageId("foo").messageBuilder(), message -> assertThat(message.getProperties().getMessageIdAsString()).isEqualTo("foo")), test(builder -> builder.properties().messageId("bar".getBytes(CHARSET)).messageBuilder(), message -> assertThat(message.getProperties().getMessageIdAsBinary()).isEqualTo("bar".getBytes(CHARSET))), test(builder -> builder.properties().messageId(TEST_UUID).messageBuilder(), message -> assertThat(message.getProperties().getMessageIdAsUuid()).isEqualTo(TEST_UUID)), test(builder -> builder.properties().correlationId(42 + 10).messageBuilder(), message -> assertThat(message.getProperties().getCorrelationIdAsLong()).isEqualTo(42 + 10)), test(builder -> builder.properties().correlationId("correlation foo").messageBuilder(), message -> assertThat(message.getProperties().getCorrelationIdAsString()).isEqualTo("correlation foo")), test(builder -> builder.properties().correlationId("correlation bar".getBytes(CHARSET)).messageBuilder(), message -> assertThat(message.getProperties().getCorrelationIdAsBinary()).isEqualTo("correlation bar".getBytes(CHARSET))), test(builder -> builder.properties().correlationId(TEST_UUID).messageBuilder(), message -> assertThat(message.getProperties().getCorrelationIdAsUuid()).isEqualTo(TEST_UUID)), test(builder -> builder, message -> assertThat(message.getProperties().getGroupSequence()).isEqualTo(-1)), test(builder -> builder.properties().groupSequence(10).messageBuilder(), message -> assertThat(message.getProperties().getGroupSequence()).isEqualTo(10)), test(builder -> builder.properties().groupSequence((long) Integer.MAX_VALUE + 10).messageBuilder(), message -> assertThat(message.getProperties().getGroupSequence()).isEqualTo((long) Integer.MAX_VALUE + 10)));
    String body = "hello";
    String userId = "yoda";
    String to = "the to address";
    String subject = "the subject";
    String replyTo = "the reply to";
    String contentType = "text/plain";
    String contentEncoding = "gzip";
    String groupId = "the group ID";
    String replyToGroupId = "the reply to group ID";
    long now = new Date().getTime();
    UUID uuid = UUID.randomUUID();
    byte[] binary = "the binary".getBytes(CHARSET);
    String string = "a string";
    String symbol = "a symbol";
    messageOperations.forEach(messageTestConfiguration -> {
        Function<MessageBuilder, MessageBuilder> messageOperation = messageTestConfiguration.messageOperation;
        Consumer<Message> messageExpectation = messageTestConfiguration.messageExpectation;
        MessageBuilder messageBuilder = codecCouple.messageBuilderSupplier.get();
        Message outboundMessage = messageOperation.apply(messageBuilder).addData(body.getBytes(CHARSET)).properties().userId(userId.getBytes(CHARSET)).to(to).subject(subject).replyTo(replyTo).contentType(contentType).contentEncoding(contentEncoding).absoluteExpiryTime(now + 1000).creationTime(now).groupId(groupId).replyToGroupId(replyToGroupId).messageBuilder().applicationProperties().entry("boolean", Boolean.FALSE).entry("byte", (byte) 1).entry("short", (short) 2).entry("int", 3).entry("long", 4l).entryUnsigned("ubyte", (byte) 1).entryUnsigned("ushort", (short) 2).entryUnsigned("uint", 3).entryUnsigned("ulong", 4l).entryUnsigned("large.ubyte", (byte) (Byte.MAX_VALUE + 10)).entryUnsigned("large.ushort", (short) (Short.MAX_VALUE + 10)).entryUnsigned("large.uint", Integer.MAX_VALUE + 10).entryUnsigned("large.ulong", Long.MAX_VALUE + 10).entry("float", 3.14f).entry("double", 6.28).entry("char", 'c').entryTimestamp("timestamp", now).entry("uuid", uuid).entry("binary", binary).entry("string", string).entrySymbol("symbol", symbol).messageBuilder().messageAnnotations().entry("annotations.boolean", Boolean.FALSE).entry("annotations.byte", (byte) 1).entry("annotations.short", (short) 2).entry("annotations.int", 3).entry("annotations.long", 4l).entryUnsigned("annotations.ubyte", (byte) 1).entryUnsigned("annotations.ushort", (short) 2).entryUnsigned("annotations.uint", 3).entryUnsigned("annotations.ulong", 4l).entryUnsigned("annotations.large.ubyte", (byte) (Byte.MAX_VALUE + 10)).entryUnsigned("annotations.large.ushort", (short) (Short.MAX_VALUE + 10)).entryUnsigned("annotations.large.uint", Integer.MAX_VALUE + 10).entryUnsigned("annotations.large.ulong", Long.MAX_VALUE + 10).entry("annotations.float", 3.14f).entry("annotations.double", 6.28).entry("annotations.char", 'c').entryTimestamp("annotations.timestamp", now).entry("annotations.uuid", uuid).entry("annotations.binary", binary).entry("annotations.string", string).entrySymbol("annotations.symbol", symbol).messageBuilder().build();
        Codec.EncodedMessage encoded = serializer.encode(outboundMessage);
        byte[] encodedData = new byte[encoded.getSize()];
        System.arraycopy(encoded.getData(), 0, encodedData, 0, encoded.getSize());
        Message inboundMessage = deserializer.decode(encodedData);
        messageExpectation.accept(inboundMessage);
        assertThat(new String(inboundMessage.getBodyAsBinary())).isEqualTo(body);
        assertThat(inboundMessage.getProperties().getUserId()).isEqualTo(userId.getBytes(CHARSET));
        assertThat(inboundMessage.getProperties().getTo()).isEqualTo(to);
        assertThat(inboundMessage.getProperties().getSubject()).isEqualTo(subject);
        assertThat(inboundMessage.getProperties().getReplyTo()).isEqualTo(replyTo);
        assertThat(inboundMessage.getProperties().getContentType()).isEqualTo(contentType);
        assertThat(inboundMessage.getProperties().getContentEncoding()).isEqualTo(contentEncoding);
        assertThat(inboundMessage.getProperties().getAbsoluteExpiryTime()).isEqualTo(now + 1000);
        assertThat(inboundMessage.getProperties().getCreationTime()).isEqualTo(now);
        assertThat(inboundMessage.getProperties().getGroupId()).isEqualTo(groupId);
        assertThat(inboundMessage.getProperties().getReplyToGroupId()).isEqualTo(replyToGroupId);
        // application properties
        assertThat(inboundMessage.getApplicationProperties().get("boolean")).isNotNull().isInstanceOf(Boolean.class).isEqualTo(Boolean.FALSE);
        assertThat(inboundMessage.getApplicationProperties().get("byte")).isNotNull().isInstanceOf(Byte.class).isEqualTo(Byte.valueOf((byte) 1));
        assertThat(inboundMessage.getApplicationProperties().get("short")).isNotNull().isInstanceOf(Short.class).isEqualTo(Short.valueOf((short) 2));
        assertThat(inboundMessage.getApplicationProperties().get("int")).isNotNull().isInstanceOf(Integer.class).isEqualTo(Integer.valueOf(3));
        assertThat(inboundMessage.getApplicationProperties().get("long")).isNotNull().isInstanceOf(Long.class).isEqualTo(Long.valueOf(4));
        assertThat(inboundMessage.getApplicationProperties().get("ubyte")).isNotNull().isInstanceOf(UnsignedByte.class).isEqualTo(UnsignedByte.valueOf((byte) 1));
        assertThat(inboundMessage.getApplicationProperties().get("ushort")).isNotNull().isInstanceOf(UnsignedShort.class).isEqualTo(UnsignedShort.valueOf((short) 2));
        assertThat(inboundMessage.getApplicationProperties().get("uint")).isNotNull().isInstanceOf(UnsignedInteger.class).isEqualTo(UnsignedInteger.valueOf(3));
        assertThat(inboundMessage.getApplicationProperties().get("ulong")).isNotNull().isInstanceOf(UnsignedLong.class).isEqualTo(UnsignedLong.valueOf(4));
        assertThat(inboundMessage.getApplicationProperties().get("large.ubyte")).isNotNull().isInstanceOf(UnsignedByte.class).asInstanceOf(InstanceOfAssertFactories.type(UnsignedByte.class)).extracting(v -> v.intValue()).isEqualTo(Byte.MAX_VALUE + 10);
        assertThat(inboundMessage.getApplicationProperties().get("large.ushort")).isNotNull().isInstanceOf(UnsignedShort.class).asInstanceOf(InstanceOfAssertFactories.type(UnsignedShort.class)).extracting(v -> v.intValue()).isEqualTo(Short.MAX_VALUE + 10);
        assertThat(inboundMessage.getApplicationProperties().get("large.uint")).isNotNull().isInstanceOf(UnsignedInteger.class).asInstanceOf(InstanceOfAssertFactories.type(UnsignedInteger.class)).extracting(v -> v.toString()).isEqualTo(BigInteger.valueOf((long) Integer.MAX_VALUE + 10L).toString());
        assertThat(inboundMessage.getApplicationProperties().get("large.ulong")).isNotNull().isInstanceOf(UnsignedLong.class).asInstanceOf(InstanceOfAssertFactories.type(UnsignedLong.class)).extracting(v -> v.toString()).isEqualTo(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TEN).toString());
        assertThat(inboundMessage.getApplicationProperties().get("float")).isNotNull().isInstanceOf(Float.class).isEqualTo(Float.valueOf(3.14f));
        assertThat(inboundMessage.getApplicationProperties().get("double")).isNotNull().isInstanceOf(Double.class).isEqualTo(Double.valueOf(6.28));
        assertThat(inboundMessage.getApplicationProperties().get("char")).isNotNull().isInstanceOf(Character.class).isEqualTo('c');
        assertThat(inboundMessage.getApplicationProperties().get("timestamp")).isNotNull().isInstanceOf(Long.class).isEqualTo(now);
        assertThat(inboundMessage.getApplicationProperties().get("uuid")).isNotNull().isInstanceOf(UUID.class).isEqualTo(uuid);
        assertThat(inboundMessage.getApplicationProperties().get("binary")).isNotNull().isInstanceOf(byte[].class).isEqualTo(binary);
        assertThat(inboundMessage.getApplicationProperties().get("string")).isNotNull().isInstanceOf(String.class).isEqualTo(string);
        assertThat(inboundMessage.getApplicationProperties().get("symbol")).isNotNull().isInstanceOf(String.class).isEqualTo(symbol);
        // message annotations
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.boolean")).isNotNull().isInstanceOf(Boolean.class).isEqualTo(Boolean.FALSE);
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.byte")).isNotNull().isInstanceOf(Byte.class).isEqualTo(Byte.valueOf((byte) 1));
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.short")).isNotNull().isInstanceOf(Short.class).isEqualTo(Short.valueOf((short) 2));
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.int")).isNotNull().isInstanceOf(Integer.class).isEqualTo(Integer.valueOf(3));
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.long")).isNotNull().isInstanceOf(Long.class).isEqualTo(Long.valueOf(4));
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.ubyte")).isNotNull().isInstanceOf(UnsignedByte.class).isEqualTo(UnsignedByte.valueOf((byte) 1));
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.ushort")).isNotNull().isInstanceOf(UnsignedShort.class).isEqualTo(UnsignedShort.valueOf((short) 2));
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.uint")).isNotNull().isInstanceOf(UnsignedInteger.class).isEqualTo(UnsignedInteger.valueOf(3));
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.ulong")).isNotNull().isInstanceOf(UnsignedLong.class).isEqualTo(UnsignedLong.valueOf(4));
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.large.ubyte")).isNotNull().isInstanceOf(UnsignedByte.class).asInstanceOf(InstanceOfAssertFactories.type(UnsignedByte.class)).extracting(v -> v.intValue()).isEqualTo(Byte.MAX_VALUE + 10);
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.large.ushort")).isNotNull().isInstanceOf(UnsignedShort.class).asInstanceOf(InstanceOfAssertFactories.type(UnsignedShort.class)).extracting(v -> v.intValue()).isEqualTo(Short.MAX_VALUE + 10);
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.large.uint")).isNotNull().isInstanceOf(UnsignedInteger.class).asInstanceOf(InstanceOfAssertFactories.type(UnsignedInteger.class)).extracting(v -> v.toString()).isEqualTo(BigInteger.valueOf((long) Integer.MAX_VALUE + 10L).toString());
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.large.ulong")).isNotNull().isInstanceOf(UnsignedLong.class).asInstanceOf(InstanceOfAssertFactories.type(UnsignedLong.class)).extracting(v -> v.toString()).isEqualTo(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TEN).toString());
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.float")).isNotNull().isInstanceOf(Float.class).isEqualTo(Float.valueOf(3.14f));
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.double")).isNotNull().isInstanceOf(Double.class).isEqualTo(Double.valueOf(6.28));
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.char")).isNotNull().isInstanceOf(Character.class).isEqualTo('c');
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.timestamp")).isNotNull().isInstanceOf(Long.class).isEqualTo(now);
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.uuid")).isNotNull().isInstanceOf(UUID.class).isEqualTo(uuid);
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.binary")).isNotNull().isInstanceOf(byte[].class).isEqualTo(binary);
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.string")).isNotNull().isInstanceOf(String.class).isEqualTo(string);
        assertThat(inboundMessage.getMessageAnnotations().get("annotations.symbol")).isNotNull().isInstanceOf(String.class).isEqualTo(symbol);
    });
}

Example 3 with Codec

use of com.rabbitmq.stream.Codec in project rabbitmq-stream-java-client by rabbitmq.

the class StreamPerfTest method call.

@Override
public Integer call() throws Exception {
    maybeDisplayVersion();
    maybeDisplayEnvironmentVariablesHelp();
    overridePropertiesWithEnvironmentVariables();
    Codec codec = createCodec(this.codecClass);
    ByteBufAllocator byteBufAllocator = ByteBufAllocator.DEFAULT;
    CompositeMeterRegistry meterRegistry = new CompositeMeterRegistry();
    String metricsPrefix = "rabbitmq.stream";
    this.metricsCollector = new MicrometerMetricsCollector(meterRegistry, metricsPrefix);
    Counter producerConfirm = meterRegistry.counter(metricsPrefix + ".producer_confirmed");
    Supplier<String> memoryReportSupplier;
    if (this.memoryReport) {
        long physicalMemory = Utils.physicalMemory();
        String physicalMemoryReport = physicalMemory == 0 ? "" : format(", physical memory %s (%d bytes)", Utils.formatByte(physicalMemory), physicalMemory);
        this.out.println(format("Max memory %s (%d bytes), max direct memory %s (%d bytes)%s", Utils.formatByte(Runtime.getRuntime().maxMemory()), Runtime.getRuntime().maxMemory(), Utils.formatByte(PlatformDependent.maxDirectMemory()), PlatformDependent.maxDirectMemory(), physicalMemoryReport));
        if (byteBufAllocator instanceof ByteBufAllocatorMetricProvider) {
            ByteBufAllocatorMetric allocatorMetric = ((ByteBufAllocatorMetricProvider) byteBufAllocator).metric();
            memoryReportSupplier = () -> {
                long usedHeapMemory = allocatorMetric.usedHeapMemory();
                long usedDirectMemory = allocatorMetric.usedDirectMemory();
                return format("Used heap memory %s (%d bytes), used direct memory %s (%d bytes)", Utils.formatByte(usedHeapMemory), usedHeapMemory, Utils.formatByte(usedDirectMemory), usedDirectMemory);
            };
        } else {
            memoryReportSupplier = () -> "";
        }
    } else {
        memoryReportSupplier = () -> "";
    }
    this.performanceMetrics = new DefaultPerformanceMetrics(meterRegistry, metricsPrefix, this.summaryFile, this.includeByteRates, this.confirmLatency, memoryReportSupplier, this.out);
    // we need to store a long in it
    this.messageSize = this.messageSize < 8 ? 8 : this.messageSize;
    ShutdownService shutdownService = new ShutdownService();
    try {
        MonitoringContext monitoringContext = new MonitoringContext(this.monitoringPort, meterRegistry);
        this.monitorings.forEach(m -> m.configure(monitoringContext));
        monitoringContext.start();
        shutdownService.wrap(closeStep("Closing monitoring context", monitoringContext::close));
        Runtime.getRuntime().addShutdownHook(new Thread(() -> shutdownService.close()));
        // FIXME add confirm latency
        ScheduledExecutorService envExecutor = Executors.newScheduledThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), this.producers), new NamedThreadFactory("stream-perf-test-env-"));
        shutdownService.wrap(closeStep("Closing environment executor", () -> envExecutor.shutdownNow()));
        boolean tls = isTls(this.uris);
        AddressResolver addrResolver;
        if (loadBalancer) {
            int defaultPort = tls ? Client.DEFAULT_TLS_PORT : Client.DEFAULT_PORT;
            List<Address> addresses = this.uris.stream().map(uri -> {
                try {
                    return new URI(uri);
                } catch (URISyntaxException e) {
                    throw new IllegalArgumentException("Error while parsing URI " + uri + ": " + e.getMessage());
                }
            }).map(uriItem -> new Address(uriItem.getHost() == null ? "localhost" : uriItem.getHost(), uriItem.getPort() == -1 ? defaultPort : uriItem.getPort())).collect(Collectors.toList());
            AtomicInteger connectionAttemptCount = new AtomicInteger(0);
            addrResolver = address -> addresses.get(connectionAttemptCount.getAndIncrement() % addresses.size());
        } else {
            if (this.addressResolver == null) {
                addrResolver = address -> address;
            } else {
                // should happen only in tests
                addrResolver = this.addressResolver;
            }
        }
        EnvironmentBuilder environmentBuilder = Environment.builder().id("stream-perf-test").uris(this.uris).addressResolver(addrResolver).scheduledExecutorService(envExecutor).metricsCollector(metricsCollector).byteBufAllocator(byteBufAllocator).codec(codec).maxProducersByConnection(this.producersByConnection).maxTrackingConsumersByConnection(this.trackingConsumersByConnection).maxConsumersByConnection(this.consumersByConnection).rpcTimeout(Duration.ofSeconds(this.rpcTimeout));
        ChannelCustomizer channelCustomizer = channel -> {
        };
        if (tls) {
            TlsConfiguration tlsConfiguration = environmentBuilder.tls();
            tlsConfiguration = tlsConfiguration.sslContext(SslContextBuilder.forClient().trustManager(Utils.TRUST_EVERYTHING_TRUST_MANAGER).build());
            environmentBuilder = tlsConfiguration.environmentBuilder();
            if (!this.sniServerNames.isEmpty()) {
                channelCustomizer = channelCustomizer.andThen(ch -> {
                    SslHandler sslHandler = ch.pipeline().get(SslHandler.class);
                    if (sslHandler != null) {
                        SSLParameters sslParameters = sslHandler.engine().getSSLParameters();
                        sslParameters.setServerNames(this.sniServerNames);
                        sslHandler.engine().setSSLParameters(sslParameters);
                    }
                });
            }
        }
        Environment environment = environmentBuilder.channelCustomizer(channelCustomizer).build();
        shutdownService.wrap(closeStep("Closing environment(s)", () -> environment.close()));
        streams = Utils.streams(this.streamCount, this.streams);
        for (String stream : streams) {
            StreamCreator streamCreator = environment.streamCreator().stream(stream).maxLengthBytes(this.maxLengthBytes).maxSegmentSizeBytes(this.maxSegmentSize).leaderLocator(this.leaderLocator);
            if (this.maxAge != null) {
                streamCreator.maxAge(this.maxAge);
            }
            try {
                streamCreator.create();
            } catch (StreamException e) {
                if (e.getCode() == Constants.RESPONSE_CODE_PRECONDITION_FAILED) {
                    String message = String.format("Warning: stream '%s' already exists, but with different properties than " + "max-length-bytes=%s, stream-max-segment-size-bytes=%s, queue-leader-locator=%s", stream, this.maxLengthBytes, this.maxSegmentSize, this.leaderLocator);
                    if (this.maxAge != null) {
                        message += String.format(", max-age=%s", this.maxAge);
                    }
                    this.out.println(message);
                } else {
                    throw e;
                }
            }
        }
        if (this.deleteStreams) {
            shutdownService.wrap(closeStep("Deleting stream(s)", () -> {
                for (String stream : streams) {
                    LOGGER.debug("Deleting {}", stream);
                    try {
                        environment.deleteStream(stream);
                        LOGGER.debug("Deleted {}", stream);
                    } catch (Exception e) {
                        LOGGER.warn("Could not delete stream {}: {}", stream, e.getMessage());
                    }
                }
            }));
        }
        List<Producer> producers = Collections.synchronizedList(new ArrayList<>(this.producers));
        List<Runnable> producerRunnables = IntStream.range(0, this.producers).mapToObj(i -> {
            Runnable rateLimiterCallback;
            if (this.rate > 0) {
                RateLimiter rateLimiter = RateLimiter.create(this.rate);
                rateLimiterCallback = () -> rateLimiter.acquire(1);
            } else {
                rateLimiterCallback = () -> {
                };
            }
            String stream = stream(this.streams, i);
            ProducerBuilder producerBuilder = environment.producerBuilder();
            String producerName = this.producerNameStrategy.apply(stream, i + 1);
            if (producerName != null && !producerName.trim().isEmpty()) {
                producerBuilder = producerBuilder.name(producerName).confirmTimeout(Duration.ZERO);
            }
            Producer producer = producerBuilder.subEntrySize(this.subEntrySize).batchSize(this.batchSize).compression(this.compression == Compression.NONE ? null : this.compression).maxUnconfirmedMessages(this.confirms).stream(stream).build();
            AtomicLong messageCount = new AtomicLong(0);
            ConfirmationHandler confirmationHandler;
            if (this.confirmLatency) {
                final PerformanceMetrics metrics = this.performanceMetrics;
                final int divisor = Utils.downSamplingDivisor(this.rate);
                confirmationHandler = confirmationStatus -> {
                    if (confirmationStatus.isConfirmed()) {
                        producerConfirm.increment();
                        // this should not affect the metric much
                        if (messageCount.incrementAndGet() % divisor == 0) {
                            try {
                                long time = Utils.readLong(confirmationStatus.getMessage().getBodyAsBinary());
                                // see below why we use current time to measure latency
                                metrics.confirmLatency(System.currentTimeMillis() - time, TimeUnit.MILLISECONDS);
                            } catch (Exception e) {
                            // not able to read the body, something wrong?
                            }
                        }
                    }
                };
            } else {
                confirmationHandler = confirmationStatus -> {
                    if (confirmationStatus.isConfirmed()) {
                        producerConfirm.increment();
                    }
                };
            }
            producers.add(producer);
            return (Runnable) () -> {
                final int msgSize = this.messageSize;
                try {
                    while (true && !Thread.currentThread().isInterrupted()) {
                        rateLimiterCallback.run();
                        // Using current time for interoperability with other tools
                        // and also across different processes.
                        // This is good enough to measure duration/latency this way
                        // in a performance tool.
                        long creationTime = System.currentTimeMillis();
                        byte[] payload = new byte[msgSize];
                        Utils.writeLong(payload, creationTime);
                        producer.send(producer.messageBuilder().addData(payload).build(), confirmationHandler);
                    }
                } catch (Exception e) {
                    if (e instanceof InterruptedException || (e.getCause() != null && e.getCause() instanceof InterruptedException)) {
                        LOGGER.info("Publisher #{} thread interrupted", i, e);
                    } else {
                        LOGGER.warn("Publisher #{} crashed", i, e);
                    }
                }
            };
        }).collect(Collectors.toList());
        List<Consumer> consumers = Collections.synchronizedList(IntStream.range(0, this.consumers).mapToObj(i -> {
            final PerformanceMetrics metrics = this.performanceMetrics;
            AtomicLong messageCount = new AtomicLong(0);
            String stream = stream(streams, i);
            ConsumerBuilder consumerBuilder = environment.consumerBuilder();
            consumerBuilder = consumerBuilder.stream(stream).offset(this.offset);
            if (this.storeEvery > 0) {
                String consumerName = this.consumerNameStrategy.apply(stream, i + 1);
                consumerBuilder = consumerBuilder.name(consumerName).autoTrackingStrategy().messageCountBeforeStorage(this.storeEvery).builder();
            }
            // we assume the publishing rate is the same order as the consuming rate
            // we actually don't want to downsample for low rates
            final int divisor = Utils.downSamplingDivisor(this.rate);
            consumerBuilder = consumerBuilder.messageHandler((context, message) -> {
                // this should not affect the metric much
                if (messageCount.incrementAndGet() % 100 == divisor) {
                    try {
                        long time = Utils.readLong(message.getBodyAsBinary());
                        // see above why we use current time to measure latency
                        metrics.latency(System.currentTimeMillis() - time, TimeUnit.MILLISECONDS);
                    } catch (Exception e) {
                    // not able to read the body, maybe not a message from the
                    // tool
                    }
                    metrics.offset(context.offset());
                }
            });
            Consumer consumer = consumerBuilder.build();
            return consumer;
        }).collect(Collectors.toList()));
        shutdownService.wrap(closeStep("Closing consumers", () -> {
            for (Consumer consumer : consumers) {
                consumer.close();
            }
        }));
        ExecutorService executorService;
        if (this.producers > 0) {
            executorService = Executors.newFixedThreadPool(this.producers, new NamedThreadFactory("stream-perf-test-publishers-"));
            for (Runnable producer : producerRunnables) {
                this.out.println("Starting producer");
                executorService.submit(producer);
            }
        } else {
            executorService = null;
        }
        shutdownService.wrap(closeStep("Closing producers", () -> {
            for (Producer p : producers) {
                p.close();
            }
        }));
        shutdownService.wrap(closeStep("Closing producers executor service", () -> {
            if (executorService != null) {
                executorService.shutdownNow();
            }
        }));
        String metricsHeader = "Arguments: " + String.join(" ", arguments);
        this.performanceMetrics.start(metricsHeader);
        shutdownService.wrap(closeStep("Closing metrics", () -> this.performanceMetrics.close()));
        CountDownLatch latch = new CountDownLatch(1);
        Runtime.getRuntime().addShutdownHook(new Thread(() -> latch.countDown()));
        try {
            latch.await();
        } catch (InterruptedException e) {
        // moving on to the closing sequence
        }
    } finally {
        shutdownService.close();
    }
    return 0;
}

Example 4 with Codec

use of com.rabbitmq.stream.Codec in project rabbitmq-stream-java-client by rabbitmq.

the class AmqpInteroperabilityTest method publishToStreamQueueConsumeFromStream.

@ParameterizedTest
@MethodSource("codecs")
void publishToStreamQueueConsumeFromStream(Codec codec) throws Exception {
    int messageCount = 10_000;
    ConnectionFactory connectionFactory = new ConnectionFactory();
    Date timestamp = new Date();
    Supplier<Stream<PropertiesTestConfiguration>> propertiesTestConfigurations = () -> Stream.of(ptc(b -> b.appId("application id"), m -> assertThat(m.getApplicationProperties().get("x-basic-app-id")).isEqualTo("application id")), ptc(b -> b.contentEncoding("content encoding"), m -> assertThat(m.getProperties().getContentEncoding()).isEqualTo("content encoding")), ptc(b -> b.contentType("content type"), m -> assertThat(m.getProperties().getContentType()).isEqualTo("content type")), ptc(b -> b.correlationId("correlation id"), m -> assertThat(m.getProperties().getCorrelationIdAsString()).isEqualTo("correlation id")), ptc(b -> b.deliveryMode(2), m -> assertThat(m.getMessageAnnotations().get("x-basic-delivery-mode")).isEqualTo(UnsignedByte.valueOf("2"))), ptc(b -> b.expiration(String.valueOf(60_000)), m -> assertThat(m.getMessageAnnotations().get("x-basic-expiration")).isEqualTo(String.valueOf(60_000))), ptc(b -> b.messageId("message id"), m -> assertThat(m.getProperties().getMessageIdAsString()).isEqualTo("message id")), ptc(b -> b.priority(5), m -> assertThat(m.getMessageAnnotations().get("x-basic-priority")).isEqualTo(UnsignedByte.valueOf("5"))), ptc(b -> b.replyTo("reply to"), m -> assertThat(m.getProperties().getReplyTo()).isEqualTo("reply to")), ptc(b -> b.timestamp(timestamp), m -> assertThat(m.getProperties().getCreationTime()).isEqualTo((timestamp.getTime() / 1000) * // in seconds in 091, in ms in 1.0, so losing some
    1000)), // precision
    ptc(b -> b.type("the type"), m -> assertThat(m.getApplicationProperties().get("x-basic-type")).isEqualTo("the type")), ptc(b -> b.userId("guest"), m -> assertThat(m.getProperties().getUserId()).isEqualTo("guest".getBytes(UTF8))));
    Supplier<Stream<HeaderTestConfiguration>> headerApplicationPropertiesTestConfigurations = () -> Stream.of(htc(h -> h.put("long.string", LongStringHelper.asLongString("long string")), ap -> assertThat(ap.get("long.string")).isEqualTo("long string")), htc(h -> h.put("short.string", "short string"), ap -> assertThat(ap.get("short.string")).isEqualTo("short string")), htc(h -> h.put("long", Long.MAX_VALUE - 1), ap -> assertThat(ap.get("long")).isEqualTo(Long.MAX_VALUE - 1)), htc(h -> h.put("byte", Byte.MAX_VALUE - 1), ap -> assertThat(ap.get("byte")).isEqualTo(Byte.MAX_VALUE - 1)), htc(h -> h.put("integer", Integer.MAX_VALUE - 1), ap -> assertThat(ap.get("integer")).isEqualTo(Integer.MAX_VALUE - 1)), htc(h -> h.put("double", Double.MAX_VALUE - 1), ap -> assertThat(ap.get("double")).isEqualTo(Double.MAX_VALUE - 1)), htc(h -> h.put("float", Float.MAX_VALUE - 1), ap -> assertThat(ap.get("float")).isEqualTo(Float.MAX_VALUE - 1)), htc(h -> h.put("boolean", Boolean.FALSE), ap -> assertThat(ap.get("boolean")).isEqualTo(Boolean.FALSE)), htc(h -> h.put("binary", "hello".getBytes(UTF8)), ap -> assertThat(ap.get("binary")).isEqualTo("hello".getBytes(UTF8))), htc(h -> h.put("timestamp", timestamp), ap -> assertThat(ap.get("timestamp")).isEqualTo((timestamp.getTime() / 1000) * // in seconds in 091, in ms in 1.0, so losing some
    1000)));
    try (Connection amqpConnection = connectionFactory.newConnection()) {
        Channel c = amqpConnection.createChannel();
        c.confirmSelect();
        Map<String, Object> headers = new HashMap<>();
        headerApplicationPropertiesTestConfigurations.get().forEach(configuration -> configuration.headerValue.accept(headers));
        AMQP.BasicProperties.Builder builder = new AMQP.BasicProperties.Builder();
        propertiesTestConfigurations.get().forEach(configuration -> configuration.builder.accept(builder));
        AMQP.BasicProperties properties = builder.headers(headers).build();
        for (int i = 0; i < messageCount; i++) {
            c.basicPublish("", stream, properties, ("amqp " + i).getBytes(UTF8));
        }
        c.waitForConfirmsOrDie(10_000);
    }
    CountDownLatch consumedLatch = new CountDownLatch(messageCount);
    Set<String> messageBodies = ConcurrentHashMap.newKeySet(messageCount);
    Set<Message> messages = ConcurrentHashMap.newKeySet(messageCount);
    Client client = cf.get(new Client.ClientParameters().codec(codec).chunkListener((client1, subscriptionId, offset, messageCount1, dataSize) -> client1.credit(subscriptionId, 1)).messageListener((subscriptionId, offset, chunkTimestamp, message) -> {
        messages.add(message);
        messageBodies.add(new String(message.getBodyAsBinary(), UTF8));
        consumedLatch.countDown();
    }));
    client.subscribe(b(1), stream, OffsetSpecification.first(), 10);
    assertThat(consumedLatch.await(10, SECONDS)).isTrue();
    assertThat(messageBodies).hasSize(messageCount);
    IntStream.range(0, messageCount).forEach(i -> assertThat(messageBodies.contains("amqp " + i)).isTrue());
    Message message = messages.iterator().next();
    propertiesTestConfigurations.get().forEach(c -> c.assertion.accept(message));
    assertThat(message.getMessageAnnotations().get("x-exchange")).isEqualTo("");
    assertThat(message.getMessageAnnotations().get("x-routing-key")).isEqualTo(stream);
    headerApplicationPropertiesTestConfigurations.get().forEach(c -> c.assertion.accept(message.getApplicationProperties()));
}

Example 5 with Codec

use of com.rabbitmq.stream.Codec in project rabbitmq-stream-java-client by rabbitmq.

the class AmqpInteroperabilityTest method publishToStreamConsumeFromStreamQueue.

@ParameterizedTest
@MethodSource("codecs")
void publishToStreamConsumeFromStreamQueue(Codec codec, TestInfo info) {
    int messageCount = 1_000;
    ConnectionFactory connectionFactory = new ConnectionFactory();
    Date timestamp = new Date();
    UUID messageIdUuid = UUID.randomUUID();
    UUID correlationIdUuid = UUID.randomUUID();
    Supplier<Stream<MessageOperation>> testMessageOperations = () -> Stream.of(mo(mb -> {
        mb.properties().messageId("the message ID");
        mb.properties().correlationId("the correlation ID");
    }, d -> {
        assertThat(d.getProperties().getMessageId()).isEqualTo("the message ID");
        assertThat(d.getProperties().getCorrelationId()).isEqualTo("the correlation ID");
    }), mo(mb -> {
        mb.properties().messageId(// larger than 091 shortstr
        StringUtils.repeat("*", 300));
        mb.properties().correlationId(// larger than 091 shortstr
        StringUtils.repeat("*", 300));
    }, d -> {
        assertThat(d.getProperties().getMessageId()).isNull();
        assertThat(d.getProperties().getCorrelationId()).isNull();
        assertThat(d.getProperties().getHeaders()).containsEntry("x-message-id", LongStringHelper.asLongString(StringUtils.repeat("*", 300))).containsEntry("x-correlation-id", LongStringHelper.asLongString(StringUtils.repeat("*", 300)));
    }), mo(mb -> {
        mb.properties().messageId(messageIdUuid);
        mb.properties().correlationId(correlationIdUuid);
    }, d -> {
        assertThat(d.getProperties().getMessageId()).isEqualTo(messageIdUuid.toString());
        assertThat(d.getProperties().getCorrelationId()).isEqualTo(correlationIdUuid.toString());
        assertThat(d.getProperties().getHeaders()).containsEntry("x-message-id-type", LongStringHelper.asLongString("uuid")).containsEntry("x-correlation-id-type", LongStringHelper.asLongString("uuid"));
    }), mo(mb -> {
        mb.properties().messageId(10);
        mb.properties().correlationId(20);
    }, d -> {
        assertThat(d.getProperties().getMessageId()).isEqualTo("10");
        assertThat(d.getProperties().getCorrelationId()).isEqualTo("20");
        assertThat(d.getProperties().getHeaders()).containsEntry("x-message-id-type", LongStringHelper.asLongString("ulong")).containsEntry("x-correlation-id-type", LongStringHelper.asLongString("ulong"));
    }), mo(mb -> {
        mb.properties().messageId("the message ID".getBytes(UTF8));
        mb.properties().correlationId("the correlation ID".getBytes(UTF8));
    }, d -> {
        assertThat(Base64.getDecoder().decode(d.getProperties().getMessageId())).isEqualTo("the message ID".getBytes(UTF8));
        assertThat(Base64.getDecoder().decode(d.getProperties().getCorrelationId())).isEqualTo("the correlation ID".getBytes(UTF8));
        assertThat(d.getProperties().getHeaders()).containsEntry("x-message-id-type", LongStringHelper.asLongString("binary")).containsEntry("x-correlation-id-type", LongStringHelper.asLongString("binary"));
    }), mo(mb -> {
        mb.properties().messageId(StringUtils.repeat("a", 300).getBytes(// larger than 091 shortstr
        UTF8));
        mb.properties().correlationId(StringUtils.repeat("b", 300).getBytes(// larger than 091 shortstr
        UTF8));
    }, d -> {
        assertThat(d.getProperties().getMessageId()).isNull();
        assertThat(d.getProperties().getCorrelationId()).isNull();
        assertThat(d.getProperties().getHeaders()).containsEntry("x-message-id", LongStringHelper.asLongString(StringUtils.repeat("a", 300).getBytes(UTF8))).containsEntry("x-correlation-id", LongStringHelper.asLongString(StringUtils.repeat("b", 300).getBytes(UTF8)));
    }));
    testMessageOperations.get().forEach(testMessageOperation -> {
        CountDownLatch confirmLatch = new CountDownLatch(messageCount);
        Client client = cf.get(new Client.ClientParameters().codec(codec).publishConfirmListener((publisherId, publishingId) -> confirmLatch.countDown()));
        String s = streamName(info);
        Client.Response response = client.create(s);
        assertThat(response.isOk()).isTrue();
        Supplier<Stream<MessageOperation>> messageOperations = () -> Stream.of(mo(mb -> mb.properties().userId("the user ID".getBytes(UTF8)), d -> assertThat(d.getProperties().getUserId()).isEqualTo("the user ID")), mo(mb -> mb.properties().to("the to address"), d -> {
        }), mo(mb -> mb.properties().subject("the subject"), d -> {
        }), mo(mb -> mb.properties().replyTo("the reply to address"), d -> assertThat(d.getProperties().getReplyTo()).isEqualTo("the reply to address")), mo(mb -> mb.properties().contentType("the content type"), d -> assertThat(d.getProperties().getContentType()).isEqualTo("the content type")), mo(mb -> mb.properties().contentEncoding("the content encoding"), d -> assertThat(d.getProperties().getContentEncoding()).isEqualTo("the content encoding")), mo(mb -> mb.properties().absoluteExpiryTime(timestamp.getTime() + 1000), d -> {
        }), mo(mb -> mb.properties().creationTime(timestamp.getTime()), d -> assertThat(d.getProperties().getTimestamp().getTime()).isEqualTo((timestamp.getTime() / 1000) * // in seconds in 091, in ms in 1.0, so losing
        1000)), mo(mb -> mb.properties().groupId("the group ID"), d -> {
        }), mo(mb -> mb.properties().groupSequence(10), d -> {
        }), mo(mb -> mb.properties().replyToGroupId("the reply to group ID"), d -> {
        }), mo(mb -> mb.applicationProperties().entry("byte", Byte.MAX_VALUE), d -> assertThat(d.getProperties().getHeaders()).containsEntry("byte", Byte.MAX_VALUE)), mo(mb -> mb.applicationProperties().entry("short", Short.MAX_VALUE), d -> assertThat(d.getProperties().getHeaders()).containsEntry("short", Short.MAX_VALUE)), mo(mb -> mb.applicationProperties().entry("integer", Integer.MAX_VALUE), d -> assertThat(d.getProperties().getHeaders()).containsEntry("integer", Integer.MAX_VALUE)), mo(mb -> mb.applicationProperties().entry("long", Long.MAX_VALUE), d -> assertThat(d.getProperties().getHeaders()).containsEntry("long", Long.MAX_VALUE)), mo(mb -> mb.applicationProperties().entry("string", "a string"), d -> assertThat(d.getProperties().getHeaders()).containsEntry("string", LongStringHelper.asLongString("a string"))), mo(mb -> mb.applicationProperties().entryTimestamp("timestamp", timestamp.getTime()), d -> assertThat(d.getProperties().getHeaders()).containsEntry("timestamp", new Date((timestamp.getTime() / 1000) * 1000))), mo(mb -> mb.applicationProperties().entry("boolean", Boolean.TRUE), d -> assertThat(d.getProperties().getHeaders()).containsEntry("boolean", Boolean.TRUE)), mo(mb -> mb.applicationProperties().entry("float", 3.14f), d -> assertThat(d.getProperties().getHeaders()).containsEntry("float", 3.14f)), mo(mb -> mb.applicationProperties().entry("binary", "hello".getBytes(UTF8)), d -> assertThat(d.getProperties().getHeaders()).containsEntry("binary", "hello".getBytes(UTF8))));
        client.declarePublisher(b(1), null, s);
        IntStream.range(0, messageCount).forEach(i -> {
            MessageBuilder messageBuilder = client.messageBuilder();
            messageBuilder.addData(("stream " + i).getBytes(UTF8));
            testMessageOperation.messageBuilderConsumer.accept(messageBuilder);
            messageOperations.get().forEach(messageOperation -> messageOperation.messageBuilderConsumer.accept(messageBuilder));
            client.publish(b(1), Collections.singletonList(messageBuilder.build()));
        });
        try (Connection c = connectionFactory.newConnection()) {
            assertThat(confirmLatch.await(10, SECONDS)).isTrue();
            Channel ch = c.createChannel();
            ch.basicQos(200);
            CountDownLatch consumedLatch = new CountDownLatch(messageCount);
            Set<String> messageBodies = ConcurrentHashMap.newKeySet(messageCount);
            Set<Delivery> messages = ConcurrentHashMap.newKeySet(messageCount);
            ch.basicConsume(s, false, Collections.singletonMap("x-stream-offset", 0), (consumerTag, message) -> {
                messages.add(message);
                messageBodies.add(new String(message.getBody(), UTF8));
                consumedLatch.countDown();
                ch.basicAck(message.getEnvelope().getDeliveryTag(), false);
            }, consumerTag -> {
            });
            assertThat(consumedLatch.await(10, SECONDS)).isTrue();
            assertThat(messageBodies).hasSize(messageCount);
            IntStream.range(0, messageCount).forEach(i -> assertThat(messageBodies.contains("stream " + i)).isTrue());
            Delivery message = messages.iterator().next();
            assertThat(message.getEnvelope().getExchange()).isEmpty();
            assertThat(message.getEnvelope().getRoutingKey()).isEqualTo(s);
            assertThat(message.getProperties().getHeaders()).containsKey("x-stream-offset");
            testMessageOperation.deliveryConsumer.accept(message);
            messageOperations.get().forEach(messageOperation -> messageOperation.deliveryConsumer.accept(message));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        response = client.delete(s);
        assertThat(response.isOk()).isTrue();
    });
}