Examples with HealthStatisticLogger org.eclipse.californium.core.network.interceptors.HealthStatisticLogger used on opensource projects

Example 1 with HealthStatisticLogger

use of org.eclipse.californium.core.network.interceptors.HealthStatisticLogger in project californium by eclipse.

the class MulticastTestClient method main.

public static void main(String[] args) {
    Configuration config = Configuration.createWithFile(CONFIG_FILE, CONFIG_HEADER, DEFAULTS);
    int unicastPort = config.get(CoapConfig.COAP_PORT);
    int multicastPort = unicastPort;
    switch(args.length) {
        default:
            System.out.println("usage: MulticastTestClient [unicast-port [multicast-port]]");
        case 2:
            multicastPort = Integer.parseInt(args[1]);
        case 1:
            unicastPort = Integer.parseInt(args[0]);
        case 0:
    }
    HealthStatisticLogger health = new HealthStatisticLogger("multicast-client", true);
    CoapEndpoint endpoint = new CoapEndpoint.Builder().setConfiguration(config).build();
    endpoint.addPostProcessInterceptor(health);
    CoapClient client = new CoapClient();
    client.setEndpoint(endpoint);
    long leisureMillis = config.get(CoapConfig.LEISURE, TimeUnit.MILLISECONDS);
    String resourcePath = "helloWorld";
    try {
        // sends an uni-cast request
        get(client, unicastPort, resourcePath);
        // sends a multicast IPv4 request
        mget(client, multicastPort, resourcePath, MulticastMode.IPv4, leisureMillis);
        // sends a broadcast IPv4 request
        mget(client, multicastPort, resourcePath, MulticastMode.IPv4_BROADCAST, leisureMillis);
        // https://bugs.openjdk.java.net/browse/JDK-8210493
        // link-local multicast is broken
        // sends a link-multicast IPv6 request
        mget(client, multicastPort, resourcePath, MulticastMode.IPv6_LINK, leisureMillis);
        // sends a site-multicast IPv6 request
        mget(client, multicastPort, resourcePath, MulticastMode.Ipv6_SITE, leisureMillis);
    } catch (ConnectorException | IOException e) {
        System.err.println("Error occurred while sending request: " + e);
    }
    health.dump();
    client.shutdown();
}

Example 2 with HealthStatisticLogger

use of org.eclipse.californium.core.network.interceptors.HealthStatisticLogger in project californium by eclipse.

the class NatTestHelper method createClientEndpoint.

CoapEndpoint createClientEndpoint(Integer cidLength) throws IOException {
    String tag = "client";
    int size = clientEndpoints.size();
    if (size > 0) {
        tag += "." + size;
    }
    DtlsHealthLogger health = new DtlsHealthLogger(tag);
    this.clientStatistics.add(health);
    // prepare secure client endpoint
    DtlsConnectorConfig clientDtlsConfig = DtlsConnectorConfig.builder(config).set(DtlsConfig.DTLS_MAX_CONNECTIONS, 20).set(DtlsConfig.DTLS_RECEIVER_THREAD_COUNT, 2).set(DtlsConfig.DTLS_CONNECTOR_THREAD_COUNT, 2).set(DtlsConfig.DTLS_CONNECTION_ID_LENGTH, cidLength).setAddress(TestTools.LOCALHOST_EPHEMERAL).setLoggingTag(tag).setHealthHandler(health).setAsList(DtlsConfig.DTLS_CIPHER_SUITES, CipherSuite.TLS_PSK_WITH_AES_128_CCM_8).setAdvancedPskStore(new AdvancedSinglePskStore(IDENITITY + "." + size, KEY.getBytes())).build();
    DebugConnectionStore clientConnectionStore = ConnectorHelper.createDebugConnectionStore(clientDtlsConfig);
    DTLSConnector clientConnector = new MyDtlsConnector(clientDtlsConfig, clientConnectionStore);
    clientConnector.setAlertHandler(new MyAlertHandler(clientDtlsConfig.getLoggingTag()));
    CoapEndpoint.Builder builder = new CoapEndpoint.Builder();
    builder.setConnector(clientConnector);
    builder.setConfiguration(config);
    CoapEndpoint clientEndpoint = builder.build();
    HealthStatisticLogger healthLogger = new HealthStatisticLogger(tag, true);
    clientCoapStatistics.add(healthLogger);
    clientEndpoint.addPostProcessInterceptor(healthLogger);
    clientEndpoint.start();
    clientConnections.add(clientConnectionStore);
    clientEndpoints.add(clientEndpoint);
    return clientEndpoint;
}

Example 3 with HealthStatisticLogger

use of org.eclipse.californium.core.network.interceptors.HealthStatisticLogger in project californium by eclipse.

the class AbstractTestServer method addLogger.

public void addLogger(boolean messageTracer) {
    // add special interceptor for message traces
    for (Endpoint ep : getEndpoints()) {
        URI uri = ep.getUri();
        String scheme = uri.getScheme();
        if (messageTracer) {
            ep.addInterceptor(new MessageTracer());
            // Anonymized IoT metrics for validation. On success, remove the OriginTracer.
            ep.addInterceptor(new AnonymizedOriginTracer(uri.getPort() + "-" + scheme));
        }
        if (ep.getPostProcessInterceptors().isEmpty()) {
            long healthStatusIntervalMillis = ep.getConfig().get(SystemConfig.HEALTH_STATUS_INTERVAL, TimeUnit.MILLISECONDS);
            if (healthStatusIntervalMillis > 0) {
                final HealthStatisticLogger healthLogger = new HealthStatisticLogger(uri.toASCIIString(), CoAP.isUdpScheme(scheme));
                if (healthLogger.isEnabled()) {
                    ep.addPostProcessInterceptor(healthLogger);
                    add(healthLogger);
                }
            }
        }
    }
}

Example 4 with HealthStatisticLogger

use of org.eclipse.californium.core.network.interceptors.HealthStatisticLogger in project californium by eclipse.

the class ExtendedTestServer method addClusterEndpoint.

private void addClusterEndpoint(ScheduledExecutorService secondaryExecutor, InetSocketAddress dtlsInterface, int nodeId, DtlsClusterConnectorConfig clusterConfiguration, ClusterNodesProvider nodesProvider, ClusterNodesDiscover nodesDiscoverer, BaseConfig cliConfig) {
    if (nodesDiscoverer == null ^ nodesProvider != null) {
        throw new IllegalArgumentException("either nodes-provider or -dicoverer is required!");
    }
    InterfaceType interfaceType = dtlsInterface.getAddress().isLoopbackAddress() ? InterfaceType.LOCAL : InterfaceType.EXTERNAL;
    Configuration configuration = getConfig(Protocol.DTLS, interfaceType);
    String tag = "dtls:node-" + nodeId + ":" + StringUtil.toString(dtlsInterface);
    int handshakeResultDelayMillis = configuration.getTimeAsInt(DTLS_HANDSHAKE_RESULT_DELAY, TimeUnit.MILLISECONDS);
    long healthStatusIntervalMillis = configuration.get(SystemConfig.HEALTH_STATUS_INTERVAL, TimeUnit.MILLISECONDS);
    Integer cidLength = configuration.get(DtlsConfig.DTLS_CONNECTION_ID_LENGTH);
    if (cidLength == null || cidLength < 6) {
        throw new IllegalArgumentException("cid length must be at least 6 for cluster!");
    }
    initCredentials();
    DtlsConnectorConfig.Builder dtlsConfigBuilder = DtlsConnectorConfig.builder(configuration);
    if (cliConfig.clientAuth != null) {
        dtlsConfigBuilder.set(DtlsConfig.DTLS_CLIENT_AUTHENTICATION_MODE, cliConfig.clientAuth);
    }
    // set node-id in dtls-config-builder's Configuration clone
    dtlsConfigBuilder.set(DtlsConfig.DTLS_CONNECTION_ID_NODE_ID, nodeId);
    AsyncAdvancedPskStore asyncPskStore = new AsyncAdvancedPskStore(new PlugPskStore());
    asyncPskStore.setDelay(handshakeResultDelayMillis);
    dtlsConfigBuilder.setAdvancedPskStore(asyncPskStore);
    dtlsConfigBuilder.setAddress(dtlsInterface);
    X509KeyManager keyManager = SslContextUtil.getX509KeyManager(serverCredentials);
    AsyncKeyManagerCertificateProvider certificateProvider = new AsyncKeyManagerCertificateProvider(keyManager, CertificateType.RAW_PUBLIC_KEY, CertificateType.X_509);
    certificateProvider.setDelay(handshakeResultDelayMillis);
    dtlsConfigBuilder.setCertificateIdentityProvider(certificateProvider);
    AsyncNewAdvancedCertificateVerifier.Builder verifierBuilder = AsyncNewAdvancedCertificateVerifier.builder();
    if (cliConfig.trustall) {
        verifierBuilder.setTrustAllCertificates();
    } else {
        verifierBuilder.setTrustedCertificates(trustedCertificates);
    }
    verifierBuilder.setTrustAllRPKs();
    AsyncNewAdvancedCertificateVerifier verifier = verifierBuilder.build();
    verifier.setDelay(handshakeResultDelayMillis);
    dtlsConfigBuilder.setAdvancedCertificateVerifier(verifier);
    dtlsConfigBuilder.setConnectionListener(new MdcConnectionListener());
    dtlsConfigBuilder.setLoggingTag(tag);
    if (healthStatusIntervalMillis > 0) {
        DtlsClusterHealthLogger health = new DtlsClusterHealthLogger(tag);
        dtlsConfigBuilder.setHealthHandler(health);
        add(health);
        // reset to prevent active logger
        dtlsConfigBuilder.set(SystemConfig.HEALTH_STATUS_INTERVAL, 0, TimeUnit.MILLISECONDS);
    }
    DtlsConnectorConfig dtlsConnectorConfig = dtlsConfigBuilder.build();
    CoapEndpoint.Builder builder = new CoapEndpoint.Builder();
    EndpointObserver endpointObserver = null;
    if (nodesDiscoverer != null) {
        DtlsManagedClusterConnector connector = new DtlsManagedClusterConnector(dtlsConnectorConfig, clusterConfiguration);
        final DtlsClusterManager manager = new DtlsClusterManager(connector, dtlsConnectorConfig.getConfiguration(), nodesDiscoverer, secondaryExecutor);
        builder.setConnector(connector);
        endpointObserver = new EndpointObserver() {

            @Override
            public void stopped(Endpoint endpoint) {
                manager.stop();
            }

            @Override
            public void started(Endpoint endpoint) {
                manager.start();
            }

            @Override
            public void destroyed(Endpoint endpoint) {
                manager.stop();
            }
        };
        components.add(manager);
    } else if (nodesProvider != null) {
        builder.setConnector(new DtlsClusterConnector(dtlsConnectorConfig, clusterConfiguration, nodesProvider));
    }
    // use dtls-config-builder's Configuration clone with the set node-id
    builder.setConfiguration(dtlsConnectorConfig.getConfiguration());
    CoapEndpoint endpoint = builder.build();
    if (healthStatusIntervalMillis > 0) {
        HealthStatisticLogger healthLogger = new HealthStatisticLogger(CoAP.COAP_SECURE_URI_SCHEME + "-" + nodeId, true);
        if (healthLogger.isEnabled()) {
            endpoint.addPostProcessInterceptor(healthLogger);
            add(healthLogger);
        }
    }
    if (endpointObserver != null) {
        endpoint.addObserver(endpointObserver);
    }
    addEndpoint(endpoint);
    print(endpoint, interfaceType);
}

Example 5 with HealthStatisticLogger

use of org.eclipse.californium.core.network.interceptors.HealthStatisticLogger in project californium by eclipse.

the class BenchmarkClient method main.

public static void main(String[] args) throws InterruptedException, IOException {
    TcpConfig.register();
    BenchmarkClient.args = args;
    startManagamentStatistic();
    config.configurationHeader = CONFIG_HEADER;
    config.customConfigurationDefaultsProvider = DEFAULTS;
    config.configurationFile = CONFIG_FILE;
    ClientInitializer.init(args, config, false);
    if (config.helpRequested) {
        System.exit(0);
    }
    // random part of PSK identity
    final SecureRandom random = new SecureRandom();
    final byte[] id = new byte[8];
    final int clients = config.clients;
    BenchmarkClient.clients = clients;
    int reverseResponses = 0;
    int notifies = 0;
    if (config.blockwiseOptions != null) {
        if (config.blockwiseOptions.bertBlocks != null && config.blockwiseOptions.bertBlocks > 0) {
            config.configuration.set(CoapConfig.MAX_MESSAGE_SIZE, 1024);
            config.configuration.set(CoapConfig.PREFERRED_BLOCK_SIZE, 1024);
            config.configuration.set(CoapConfig.TCP_NUMBER_OF_BULK_BLOCKS, config.blockwiseOptions.bertBlocks);
        } else if (config.blockwiseOptions.blocksize != null) {
            config.configuration.set(CoapConfig.MAX_MESSAGE_SIZE, config.blockwiseOptions.blocksize);
            config.configuration.set(CoapConfig.PREFERRED_BLOCK_SIZE, config.blockwiseOptions.blocksize);
        }
    }
    if (config.reverse != null) {
        reverseResponses = config.reverse.responses;
    }
    if (config.observe != null) {
        notifies = config.observe.notifies;
    }
    if (config.timeout != null) {
        config.configuration.set(CoapConfig.NON_LIFETIME, config.timeout, TimeUnit.MILLISECONDS);
    }
    offload = config.configuration.get(CoapConfig.USE_MESSAGE_OFFLOADING);
    URI tempUri;
    try {
        tempUri = new URI(config.uri);
    } catch (URISyntaxException e) {
        tempUri = null;
        System.err.println("Invalid URI: " + e.getMessage());
        System.exit(-1);
    }
    final URI uri = tempUri;
    overallRequests = (config.requests * clients);
    overallReverseResponses = (reverseResponses * clients);
    overallNotifies = (notifies * clients);
    if (overallRequests < 0) {
        // overflow
        overallRequests = Integer.MAX_VALUE;
    }
    if (overallReverseResponses < 0) {
        // overflow
        overallReverseResponses = Integer.MAX_VALUE;
    }
    if (overallNotifies < 0) {
        // overflow
        overallNotifies = Integer.MAX_VALUE;
    }
    overallRequestsDownCounter.set(overallRequests);
    overallResponsesDownCounter.set(overallRequests);
    overallReverseResponsesDownCounter.set(overallReverseResponses);
    overallNotifiesDownCounter.set(overallNotifies);
    final List<BenchmarkClient> clientList = Collections.synchronizedList(new ArrayList<BenchmarkClient>(clients));
    ScheduledExecutorService executor = ExecutorsUtil.newScheduledThreadPool(Runtime.getRuntime().availableProcessors(), new DaemonThreadFactory("Aux#"));
    final ScheduledExecutorService connectorExecutor = config.configuration.get(BENCHMARK_CLIENT_THREADS) == 0 ? executor : null;
    final boolean secure = CoAP.isSecureScheme(uri.getScheme());
    final boolean dtls = secure && !CoAP.isTcpScheme(uri.getScheme());
    final ScheduledThreadPoolExecutor secondaryExecutor = new ScheduledThreadPoolExecutor(2, new DaemonThreadFactory("Aux(secondary)#"));
    String proxyMessage = "";
    if (config.proxy != null) {
        proxyMessage = "via proxy " + config.proxy + " ";
    }
    System.out.format("Create %d %s%sbenchmark clients, expect to send %d requests overall %sto %s%n", clients, !config.stop ? "none-stop " : "", secure ? "secure " : "", overallRequests, proxyMessage, uri);
    if (overallReverseResponses > 0) {
        if (config.reverse.min.equals(config.reverse.max)) {
            System.out.format("Expect %d notifies sent, interval %d [ms]%n", overallReverseResponses, config.reverse.min);
        } else {
            System.out.format("Expect %d notifies sent, interval %d ... %d [ms]%n", overallReverseResponses, config.reverse.min, config.reverse.max);
        }
    }
    if (overallNotifies > 0) {
        System.out.format("Expect %d notifies received, reregister every %d and register every %d notify%n", overallNotifies, config.observe.reregister, config.observe.register);
    }
    initialConnectDownCounter.set(clients);
    final boolean psk = config.authenticationModes.contains(AuthenticationMode.PSK) || config.authenticationModes.contains(AuthenticationMode.ECDHE_PSK);
    final boolean rpk = config.authenticationModes.contains(AuthenticationMode.RPK);
    long startupNanos = System.nanoTime();
    final AuthenticationMode authentication = config.authenticationModes.isEmpty() ? null : config.authenticationModes.get(0);
    final CountDownLatch start = new CountDownLatch(clients);
    if (secure && authentication != null) {
        switch(authentication) {
            case NONE:
                System.out.println("No authentication.");
                break;
            case PSK:
                System.out.println("Use PSK.");
                break;
            case RPK:
                System.out.println("Use RPK.");
                break;
            case X509:
                System.out.println("Use X509.");
                break;
            case ECDHE_PSK:
                System.out.println("Use PSK/ECDHE.");
                break;
        }
    }
    final ThreadLocalKeyPairGenerator keyPairGenerator = rpk ? createKeyPairGenerator() : null;
    // Create & start clients
    final AtomicBoolean errors = new AtomicBoolean();
    health = new HealthStatisticLogger(uri.getScheme(), CoAP.isUdpScheme(uri.getScheme()));
    if (CoAP.isUdpScheme(uri.getScheme())) {
        netstat4 = new NetStatLogger("udp4", false);
        netstat6 = new NetStatLogger("udp6", true);
    }
    final String tag = config.tag == null ? "client-" : config.tag;
    final int pskOffset = config.pskIndex != null ? config.pskIndex : 0;
    for (int index = 0; index < clients; ++index) {
        final int currentIndex = index;
        final String identity;
        final byte[] secret;
        if (secure && psk) {
            if (config.pskStore != null) {
                int pskIndex = (pskOffset + index) % config.pskStore.size();
                identity = config.pskStore.getIdentity(pskIndex);
                secret = config.pskStore.getSecrets(pskIndex);
            } else if (index == 0) {
                identity = config.identity;
                secret = config.secretKey;
            } else {
                random.nextBytes(id);
                identity = ConnectorConfig.PSK_IDENTITY_PREFIX + StringUtil.byteArray2Hex(id);
                secret = config.secretKey;
            }
        } else {
            identity = null;
            secret = null;
        }
        Runnable run = new Runnable() {

            @Override
            public void run() {
                if (errors.get()) {
                    return;
                }
                ClientConfig connectionConfig = config;
                if (secure) {
                    if (rpk) {
                        if (keyPairGenerator != null) {
                            try {
                                KeyPairGenerator generator = keyPairGenerator.current();
                                generator.initialize(new ECGenParameterSpec("secp256r1"), RandomManager.currentSecureRandom());
                                KeyPair keyPair = generator.generateKeyPair();
                                connectionConfig = connectionConfig.create(keyPair.getPrivate(), keyPair.getPublic());
                            } catch (GeneralSecurityException ex) {
                                if (!errors.getAndSet(true)) {
                                    ex.printStackTrace();
                                    System.out.format("Failed after %d clients, exit Benchmark.%n", (clients - start.getCount()));
                                    System.exit(-1);
                                }
                            }
                        }
                    }
                    if (psk) {
                        connectionConfig = connectionConfig.create(identity, secret);
                    }
                }
                if (connectionConfig == config) {
                    connectionConfig = config.create();
                }
                connectionConfig.tag = tag + currentIndex;
                CoapEndpoint coapEndpoint = ClientInitializer.createEndpoint(connectionConfig, connectorExecutor);
                if (health.isEnabled()) {
                    coapEndpoint.addPostProcessInterceptor(health);
                }
                BenchmarkClient client = new BenchmarkClient(currentIndex, config.reverse, uri, coapEndpoint, connectorExecutor, secondaryExecutor);
                clientList.add(client);
                try {
                    client.start();
                    start.countDown();
                    if (currentIndex == 0) {
                        // first client, so test request
                        if (client.test()) {
                            System.out.println("Benchmark clients, first request successful.");
                        } else {
                            System.out.format("Request %s POST failed, exit Benchmark.%n", uri);
                            System.exit(-1);
                        }
                        // ensure to use ephemeral port for other clients
                        config.localPort = null;
                    }
                } catch (RuntimeException e) {
                    if (!errors.getAndSet(true)) {
                        e.printStackTrace();
                        System.out.format("Failed after %d clients, exit Benchmark.%n", (clients - start.getCount()));
                        System.exit(-1);
                    }
                }
            }
        };
        if (index == 0) {
            // first client, so test request
            if (identity != null) {
                // first client, so test request
                if (secret == null) {
                    System.out.println("ID: " + identity);
                } else {
                    System.out.println("ID: " + identity + ", " + new String(secret));
                }
            }
            run.run();
        } else if (!errors.get()) {
            startupNanos = System.nanoTime();
            executor.execute(run);
        }
    }
    start.await();
    startupNanos = System.nanoTime() - startupNanos;
    if (clients == 1) {
        System.out.format("Benchmark client created. %s%n", formatTime(startupNanos));
    } else {
        System.out.format("Benchmark clients created. %s%s%n", formatTime(startupNanos), formatPerSecond("clients", clients - 1, startupNanos));
    }
    // Start Test
    boolean stale = false;
    startRequestNanos = System.nanoTime();
    startReverseResponseNanos = startRequestNanos;
    startNotifiesNanos = startRequestNanos;
    long lastRequestsCountDown = overallRequestsDownCounter.get();
    long lastResponsesCountDown = overallResponsesDownCounter.get();
    long lastTransmissions = transmissionCounter.get();
    long lastRetransmissions = retransmissionCounter.get();
    long lastTransmissionErrrors = transmissionErrorCounter.get();
    int lastUnavailable = overallServiceUnavailable.get();
    int lastHonoCmds = overallHonoCmds.get();
    int requestsPerClient = config.requests <= 10 ? config.requests : -1;
    for (int index = clients - 1; index >= 0; --index) {
        BenchmarkClient client = clientList.get(index);
        if (index == 0) {
            --requestsPerClient;
        }
        client.startBenchmark(requestsPerClient);
    }
    registerShutdown();
    System.out.println("Benchmark started.");
    long staleTime = System.nanoTime();
    long interval = config.interval == null ? 0 : TimeUnit.MILLISECONDS.toNanos(config.interval);
    if (dtls) {
        interval = Math.max(interval, DTLS_TIMEOUT_NANOS);
    }
    long staleTimeout = DEFAULT_TIMEOUT_NANOS + interval;
    int count = 0;
    // Wait with timeout or all requests send.
    while (!overallRequestsDone.await(DEFAULT_TIMEOUT_NANOS, TimeUnit.NANOSECONDS)) {
        long currentRequestsCountDown = overallRequestsDownCounter.get();
        long currentResponsesCountDown = overallResponsesDownCounter.get();
        int numberOfClients = clientCounter.get();
        int connectsPending = initialConnectDownCounter.get();
        long requestsDifference = (lastRequestsCountDown - currentRequestsCountDown);
        long responsesDifference = (lastResponsesCountDown - currentResponsesCountDown);
        long currentOverallSentRequests = overallRequests - currentRequestsCountDown;
        if ((responsesDifference == 0 && currentResponsesCountDown < overallRequests) || (numberOfClients == 0)) {
            // no new requests, clients are stale, or no clients left
            // adjust start time with timeout
            long timeout = System.nanoTime() - staleTime;
            if ((timeout - staleTimeout) > 0) {
                startRequestNanos += timeout;
                startReverseResponseNanos = startRequestNanos;
                startNotifiesNanos = startRequestNanos;
                stale = true;
                System.out.format("[%04d]: %d requests, stale (%d clients, %d pending)%n", ++count, currentOverallSentRequests, numberOfClients, connectsPending);
                break;
            }
        } else {
            staleTime = System.nanoTime();
        }
        long transmissions = transmissionCounter.get();
        long transmissionsDifference = transmissions - lastTransmissions;
        long retransmissions = retransmissionCounter.get();
        long retransmissionsDifference = retransmissions - lastRetransmissions;
        long transmissionErrors = transmissionErrorCounter.get();
        long transmissionErrorsDifference = transmissionErrors - lastTransmissionErrrors;
        int unavailable = overallServiceUnavailable.get();
        int unavailableDifference = unavailable - lastUnavailable;
        int honoCmds = overallHonoCmds.get();
        int honoCmdsDifference = honoCmds - lastHonoCmds;
        lastRequestsCountDown = currentRequestsCountDown;
        lastResponsesCountDown = currentResponsesCountDown;
        lastTransmissions = transmissions;
        lastRetransmissions = retransmissions;
        lastTransmissionErrrors = transmissionErrors;
        lastUnavailable = unavailable;
        lastHonoCmds = honoCmds;
        StringBuilder line = new StringBuilder();
        line.append(String.format("[%04d]: ", ++count));
        line.append(String.format("%d requests (%d reqs/s", currentOverallSentRequests, roundDiv(responsesDifference, DEFAULT_TIMEOUT_SECONDS)));
        line.append(", ").append(formatRetransmissions(retransmissionsDifference, transmissionsDifference, responsesDifference));
        line.append(", ").append(formatTransmissionErrors(transmissionErrorsDifference, requestsDifference, responsesDifference));
        if (unavailable > 0) {
            line.append(", ").append(formatUnavailable(unavailableDifference, responsesDifference));
        }
        if (honoCmds > 0) {
            line.append(", ").append(formatHonoCmds(honoCmdsDifference, responsesDifference));
        }
        line.append(String.format(", %d clients", numberOfClients));
        if (connectsPending > 0) {
            line.append(String.format(", %d pending", connectsPending));
        }
        line.append(")");
        System.out.println(line);
    }
    timeRequestNanos = System.nanoTime() - startRequestNanos;
    boolean observe = false;
    long lastReverseResponsesCountDown = overallReverseResponsesDownCounter.get();
    if (config.reverse != null && lastReverseResponsesCountDown > 0) {
        System.out.println("Requests send.");
        long lastChangeNanoRealtime = ClockUtil.nanoRealtime();
        while (!overallReveresResponsesDone.await(DEFAULT_TIMEOUT_NANOS, TimeUnit.NANOSECONDS)) {
            long currentReverseResponsesCountDown = overallReverseResponsesDownCounter.get();
            int numberOfClients = clientCounter.get();
            int observers = overallObserverCounter.get();
            long reverseResponsesDifference = (lastReverseResponsesCountDown - currentReverseResponsesCountDown);
            long currentOverallReverseResponses = overallReverseResponses - currentReverseResponsesCountDown;
            if (overallObservationRegistrationCounter.get() > 0) {
                observe = true;
            }
            long time = 0;
            if (currentReverseResponsesCountDown < overallReverseResponses) {
                if (reverseResponsesDifference == 0) {
                    time = ClockUtil.nanoRealtime() - lastChangeNanoRealtime;
                } else {
                    lastChangeNanoRealtime = ClockUtil.nanoRealtime();
                }
            } else {
                // wait extra DEFAULT_TIMEOUT_NANOS for start of reverse
                // responses.
                time = ClockUtil.nanoRealtime() - lastChangeNanoRealtime - DEFAULT_TIMEOUT_NANOS;
            }
            if (config.reverse.max < TimeUnit.NANOSECONDS.toMillis(time - DEFAULT_TIMEOUT_NANOS) || (numberOfClients == 0)) {
                // no new notifies for interval max, clients are stale, or
                // no clients left
                // adjust start time with timeout
                startReverseResponseNanos += time;
                stale = true;
                if (observe) {
                    System.out.format("[%04d]: %d notifies, stale (%d clients, %d observes)%n", ++count, currentOverallReverseResponses, numberOfClients, observers);
                } else {
                    System.out.format("[%04d]: %d reverse-responses, stale (%d clients)%n", ++count, currentOverallReverseResponses, numberOfClients);
                }
                break;
            }
            lastReverseResponsesCountDown = currentReverseResponsesCountDown;
            if (observe) {
                System.out.format("[%04d]: %d notifies (%d notifies/s, %d clients, %d observes)%n", ++count, currentOverallReverseResponses, roundDiv(reverseResponsesDifference, DEFAULT_TIMEOUT_SECONDS), numberOfClients, observers);
            } else {
                System.out.format("[%04d]: %d reverse-responses (%d reverse-responses/s, %d clients)%n", ++count, currentOverallReverseResponses, roundDiv(reverseResponsesDifference, DEFAULT_TIMEOUT_SECONDS), numberOfClients);
            }
        }
    }
    timeReverseResponseNanos = System.nanoTime() - startReverseResponseNanos;
    long lastNotifiesCountDown = overallNotifiesDownCounter.get();
    if (config.observe != null && lastNotifiesCountDown > 0) {
        System.out.println("Observe-Requests send.");
        long currentOverallSentRequests = overallRequests - overallRequestsDownCounter.get();
        long lastChangeNanoRealtime = ClockUtil.nanoRealtime();
        while (!overallNotifiesDone.await(DEFAULT_TIMEOUT_NANOS, TimeUnit.NANOSECONDS)) {
            long currentRequestsCountDown = overallRequestsDownCounter.get();
            long requestsDifference = (lastRequestsCountDown - currentRequestsCountDown);
            currentOverallSentRequests += requestsDifference;
            long currentNotifiesCountDown = overallNotifiesDownCounter.get();
            int numberOfClients = clientCounter.get();
            long notifiesDifference = (lastNotifiesCountDown - currentNotifiesCountDown);
            long currentOverallNotifies = overallNotifies - currentNotifiesCountDown;
            long time = 0;
            if (currentNotifiesCountDown < overallNotifies) {
                if (notifiesDifference == 0) {
                    time = ClockUtil.nanoRealtime() - lastChangeNanoRealtime;
                } else {
                    lastChangeNanoRealtime = ClockUtil.nanoRealtime();
                }
            } else {
                // wait extra DEFAULT_TIMEOUT_NANOS for start of reverse
                // responses.
                time = ClockUtil.nanoRealtime() - lastChangeNanoRealtime - DEFAULT_TIMEOUT_NANOS;
            }
            if (0 < TimeUnit.NANOSECONDS.toMillis(time - DEFAULT_TIMEOUT_NANOS) || (numberOfClients == 0)) {
                // no new notifies for interval max, clients are stale, or
                // no clients left
                // adjust start time with timeout
                startNotifiesNanos += time;
                stale = true;
                System.out.format("[%04d]: %d notifies, %d request, stale (%d clients)%n", ++count, currentOverallNotifies, currentOverallSentRequests, numberOfClients);
                break;
            }
            lastRequestsCountDown = currentRequestsCountDown;
            lastNotifiesCountDown = currentNotifiesCountDown;
            System.out.format("[%04d]: %d notifies, %d request (%d notifies/s, %d clients)%n", ++count, currentOverallNotifies, currentOverallSentRequests, roundDiv(notifiesDifference, DEFAULT_TIMEOUT_SECONDS), numberOfClients);
        }
    }
    timeNotifiesNanos = System.nanoTime() - startNotifiesNanos;
    // long overallSentReverseResponses = overallReverseResponses -
    // overallReverseResponsesDownCounter.getCount();
    System.out.format("%d benchmark clients %s.%n", clients, stale ? "stopped" : "finished");
    // stop and collect per client requests
    final int[] statistic = new int[clients];
    final CountDownLatch stop = new CountDownLatch(clients);
    for (int index = 0; index < clients; ++index) {
        final int currentIndex = index;
        Runnable run = new Runnable() {

            @Override
            public void run() {
                BenchmarkClient client = clientList.get(currentIndex);
                int requests = client.destroy();
                synchronized (statistic) {
                    statistic[currentIndex] = requests;
                }
                stop.countDown();
            }
        };
        executor.execute(run);
    }
    stop.await();
    Thread.sleep(1000);
    executor.shutdown();
    if (1 < clients) {
        synchronized (statistic) {
            Arrays.sort(statistic);
        }
        int grouped = 10;
        int last = 0;
        if (overallRequests > 500000) {
            grouped = overallRequests / 50000;
        }
        for (int index = 1; index < clients; ++index) {
            if ((statistic[index] / grouped) > (statistic[last] / grouped)) {
                System.out.println(formatClientRequests(statistic, index, last));
                last = index;
            }
        }
        System.out.println(formatClientRequests(statistic, clients, last));
    }
    done = true;
}