use of com.rabbitmq.stream.StreamCreator.LeaderLocator in project rabbitmq-stream-java-client by rabbitmq.
the class StreamPerfTest method call.
@Override
public Integer call() throws Exception {
maybeDisplayVersion();
maybeDisplayEnvironmentVariablesHelp();
overridePropertiesWithEnvironmentVariables();
Codec codec = createCodec(this.codecClass);
ByteBufAllocator byteBufAllocator = ByteBufAllocator.DEFAULT;
CompositeMeterRegistry meterRegistry = new CompositeMeterRegistry();
String metricsPrefix = "rabbitmq.stream";
this.metricsCollector = new MicrometerMetricsCollector(meterRegistry, metricsPrefix);
Counter producerConfirm = meterRegistry.counter(metricsPrefix + ".producer_confirmed");
Supplier<String> memoryReportSupplier;
if (this.memoryReport) {
long physicalMemory = Utils.physicalMemory();
String physicalMemoryReport = physicalMemory == 0 ? "" : format(", physical memory %s (%d bytes)", Utils.formatByte(physicalMemory), physicalMemory);
this.out.println(format("Max memory %s (%d bytes), max direct memory %s (%d bytes)%s", Utils.formatByte(Runtime.getRuntime().maxMemory()), Runtime.getRuntime().maxMemory(), Utils.formatByte(PlatformDependent.maxDirectMemory()), PlatformDependent.maxDirectMemory(), physicalMemoryReport));
if (byteBufAllocator instanceof ByteBufAllocatorMetricProvider) {
ByteBufAllocatorMetric allocatorMetric = ((ByteBufAllocatorMetricProvider) byteBufAllocator).metric();
memoryReportSupplier = () -> {
long usedHeapMemory = allocatorMetric.usedHeapMemory();
long usedDirectMemory = allocatorMetric.usedDirectMemory();
return format("Used heap memory %s (%d bytes), used direct memory %s (%d bytes)", Utils.formatByte(usedHeapMemory), usedHeapMemory, Utils.formatByte(usedDirectMemory), usedDirectMemory);
};
} else {
memoryReportSupplier = () -> "";
}
} else {
memoryReportSupplier = () -> "";
}
this.performanceMetrics = new DefaultPerformanceMetrics(meterRegistry, metricsPrefix, this.summaryFile, this.includeByteRates, this.confirmLatency, memoryReportSupplier, this.out);
// we need to store a long in it
this.messageSize = this.messageSize < 8 ? 8 : this.messageSize;
ShutdownService shutdownService = new ShutdownService();
try {
MonitoringContext monitoringContext = new MonitoringContext(this.monitoringPort, meterRegistry);
this.monitorings.forEach(m -> m.configure(monitoringContext));
monitoringContext.start();
shutdownService.wrap(closeStep("Closing monitoring context", monitoringContext::close));
Runtime.getRuntime().addShutdownHook(new Thread(() -> shutdownService.close()));
// FIXME add confirm latency
ScheduledExecutorService envExecutor = Executors.newScheduledThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), this.producers), new NamedThreadFactory("stream-perf-test-env-"));
shutdownService.wrap(closeStep("Closing environment executor", () -> envExecutor.shutdownNow()));
boolean tls = isTls(this.uris);
AddressResolver addrResolver;
if (loadBalancer) {
int defaultPort = tls ? Client.DEFAULT_TLS_PORT : Client.DEFAULT_PORT;
List<Address> addresses = this.uris.stream().map(uri -> {
try {
return new URI(uri);
} catch (URISyntaxException e) {
throw new IllegalArgumentException("Error while parsing URI " + uri + ": " + e.getMessage());
}
}).map(uriItem -> new Address(uriItem.getHost() == null ? "localhost" : uriItem.getHost(), uriItem.getPort() == -1 ? defaultPort : uriItem.getPort())).collect(Collectors.toList());
AtomicInteger connectionAttemptCount = new AtomicInteger(0);
addrResolver = address -> addresses.get(connectionAttemptCount.getAndIncrement() % addresses.size());
} else {
if (this.addressResolver == null) {
addrResolver = address -> address;
} else {
// should happen only in tests
addrResolver = this.addressResolver;
}
}
EnvironmentBuilder environmentBuilder = Environment.builder().id("stream-perf-test").uris(this.uris).addressResolver(addrResolver).scheduledExecutorService(envExecutor).metricsCollector(metricsCollector).byteBufAllocator(byteBufAllocator).codec(codec).maxProducersByConnection(this.producersByConnection).maxTrackingConsumersByConnection(this.trackingConsumersByConnection).maxConsumersByConnection(this.consumersByConnection).rpcTimeout(Duration.ofSeconds(this.rpcTimeout));
ChannelCustomizer channelCustomizer = channel -> {
};
if (tls) {
TlsConfiguration tlsConfiguration = environmentBuilder.tls();
tlsConfiguration = tlsConfiguration.sslContext(SslContextBuilder.forClient().trustManager(Utils.TRUST_EVERYTHING_TRUST_MANAGER).build());
environmentBuilder = tlsConfiguration.environmentBuilder();
if (!this.sniServerNames.isEmpty()) {
channelCustomizer = channelCustomizer.andThen(ch -> {
SslHandler sslHandler = ch.pipeline().get(SslHandler.class);
if (sslHandler != null) {
SSLParameters sslParameters = sslHandler.engine().getSSLParameters();
sslParameters.setServerNames(this.sniServerNames);
sslHandler.engine().setSSLParameters(sslParameters);
}
});
}
}
Environment environment = environmentBuilder.channelCustomizer(channelCustomizer).build();
shutdownService.wrap(closeStep("Closing environment(s)", () -> environment.close()));
streams = Utils.streams(this.streamCount, this.streams);
for (String stream : streams) {
StreamCreator streamCreator = environment.streamCreator().stream(stream).maxLengthBytes(this.maxLengthBytes).maxSegmentSizeBytes(this.maxSegmentSize).leaderLocator(this.leaderLocator);
if (this.maxAge != null) {
streamCreator.maxAge(this.maxAge);
}
try {
streamCreator.create();
} catch (StreamException e) {
if (e.getCode() == Constants.RESPONSE_CODE_PRECONDITION_FAILED) {
String message = String.format("Warning: stream '%s' already exists, but with different properties than " + "max-length-bytes=%s, stream-max-segment-size-bytes=%s, queue-leader-locator=%s", stream, this.maxLengthBytes, this.maxSegmentSize, this.leaderLocator);
if (this.maxAge != null) {
message += String.format(", max-age=%s", this.maxAge);
}
this.out.println(message);
} else {
throw e;
}
}
}
if (this.deleteStreams) {
shutdownService.wrap(closeStep("Deleting stream(s)", () -> {
for (String stream : streams) {
LOGGER.debug("Deleting {}", stream);
try {
environment.deleteStream(stream);
LOGGER.debug("Deleted {}", stream);
} catch (Exception e) {
LOGGER.warn("Could not delete stream {}: {}", stream, e.getMessage());
}
}
}));
}
List<Producer> producers = Collections.synchronizedList(new ArrayList<>(this.producers));
List<Runnable> producerRunnables = IntStream.range(0, this.producers).mapToObj(i -> {
Runnable rateLimiterCallback;
if (this.rate > 0) {
RateLimiter rateLimiter = RateLimiter.create(this.rate);
rateLimiterCallback = () -> rateLimiter.acquire(1);
} else {
rateLimiterCallback = () -> {
};
}
String stream = stream(this.streams, i);
ProducerBuilder producerBuilder = environment.producerBuilder();
String producerName = this.producerNameStrategy.apply(stream, i + 1);
if (producerName != null && !producerName.trim().isEmpty()) {
producerBuilder = producerBuilder.name(producerName).confirmTimeout(Duration.ZERO);
}
Producer producer = producerBuilder.subEntrySize(this.subEntrySize).batchSize(this.batchSize).compression(this.compression == Compression.NONE ? null : this.compression).maxUnconfirmedMessages(this.confirms).stream(stream).build();
AtomicLong messageCount = new AtomicLong(0);
ConfirmationHandler confirmationHandler;
if (this.confirmLatency) {
final PerformanceMetrics metrics = this.performanceMetrics;
final int divisor = Utils.downSamplingDivisor(this.rate);
confirmationHandler = confirmationStatus -> {
if (confirmationStatus.isConfirmed()) {
producerConfirm.increment();
// this should not affect the metric much
if (messageCount.incrementAndGet() % divisor == 0) {
try {
long time = Utils.readLong(confirmationStatus.getMessage().getBodyAsBinary());
// see below why we use current time to measure latency
metrics.confirmLatency(System.currentTimeMillis() - time, TimeUnit.MILLISECONDS);
} catch (Exception e) {
// not able to read the body, something wrong?
}
}
}
};
} else {
confirmationHandler = confirmationStatus -> {
if (confirmationStatus.isConfirmed()) {
producerConfirm.increment();
}
};
}
producers.add(producer);
return (Runnable) () -> {
final int msgSize = this.messageSize;
try {
while (true && !Thread.currentThread().isInterrupted()) {
rateLimiterCallback.run();
// Using current time for interoperability with other tools
// and also across different processes.
// This is good enough to measure duration/latency this way
// in a performance tool.
long creationTime = System.currentTimeMillis();
byte[] payload = new byte[msgSize];
Utils.writeLong(payload, creationTime);
producer.send(producer.messageBuilder().addData(payload).build(), confirmationHandler);
}
} catch (Exception e) {
if (e instanceof InterruptedException || (e.getCause() != null && e.getCause() instanceof InterruptedException)) {
LOGGER.info("Publisher #{} thread interrupted", i, e);
} else {
LOGGER.warn("Publisher #{} crashed", i, e);
}
}
};
}).collect(Collectors.toList());
List<Consumer> consumers = Collections.synchronizedList(IntStream.range(0, this.consumers).mapToObj(i -> {
final PerformanceMetrics metrics = this.performanceMetrics;
AtomicLong messageCount = new AtomicLong(0);
String stream = stream(streams, i);
ConsumerBuilder consumerBuilder = environment.consumerBuilder();
consumerBuilder = consumerBuilder.stream(stream).offset(this.offset);
if (this.storeEvery > 0) {
String consumerName = this.consumerNameStrategy.apply(stream, i + 1);
consumerBuilder = consumerBuilder.name(consumerName).autoTrackingStrategy().messageCountBeforeStorage(this.storeEvery).builder();
}
// we assume the publishing rate is the same order as the consuming rate
// we actually don't want to downsample for low rates
final int divisor = Utils.downSamplingDivisor(this.rate);
consumerBuilder = consumerBuilder.messageHandler((context, message) -> {
// this should not affect the metric much
if (messageCount.incrementAndGet() % 100 == divisor) {
try {
long time = Utils.readLong(message.getBodyAsBinary());
// see above why we use current time to measure latency
metrics.latency(System.currentTimeMillis() - time, TimeUnit.MILLISECONDS);
} catch (Exception e) {
// not able to read the body, maybe not a message from the
// tool
}
metrics.offset(context.offset());
}
});
Consumer consumer = consumerBuilder.build();
return consumer;
}).collect(Collectors.toList()));
shutdownService.wrap(closeStep("Closing consumers", () -> {
for (Consumer consumer : consumers) {
consumer.close();
}
}));
ExecutorService executorService;
if (this.producers > 0) {
executorService = Executors.newFixedThreadPool(this.producers, new NamedThreadFactory("stream-perf-test-publishers-"));
for (Runnable producer : producerRunnables) {
this.out.println("Starting producer");
executorService.submit(producer);
}
} else {
executorService = null;
}
shutdownService.wrap(closeStep("Closing producers", () -> {
for (Producer p : producers) {
p.close();
}
}));
shutdownService.wrap(closeStep("Closing producers executor service", () -> {
if (executorService != null) {
executorService.shutdownNow();
}
}));
String metricsHeader = "Arguments: " + String.join(" ", arguments);
this.performanceMetrics.start(metricsHeader);
shutdownService.wrap(closeStep("Closing metrics", () -> this.performanceMetrics.close()));
CountDownLatch latch = new CountDownLatch(1);
Runtime.getRuntime().addShutdownHook(new Thread(() -> latch.countDown()));
try {
latch.await();
} catch (InterruptedException e) {
// moving on to the closing sequence
}
} finally {
shutdownService.close();
}
return 0;
}
Aggregations