Examples with LatencyMetric com.linkedin.pinot.common.metrics.LatencyMetric used on opensource projects

Example 1 with LatencyMetric

use of com.linkedin.pinot.common.metrics.LatencyMetric in project pinot by linkedin.

the class AggregatedPoolStats method refresh.

public void refresh() {
    int totalCreated = 0;
    int totalDestroyed = 0;
    int totalCreateErrors = 0;
    int totalDestroyErrors = 0;
    int totalBadDestroyed = 0;
    int totalTimedOut = 0;
    int checkedOut = 0;
    int maxPoolSize = 0;
    int minPoolSize = 0;
    int poolSize = 0;
    int sampleMaxCheckedOut = 0;
    int sampleMaxPoolSize = 0;
    int idleCount = 0;
    AggregatedHistogram<Sampling> waitTimeHist = new AggregatedHistogram<Sampling>();
    AggregatedHistogram<Sampling> createTimeHist = new AggregatedHistogram<Sampling>();
    for (PoolStatsProvider p : _poolStatsProvider) {
        PoolStats<T> s = p.getStats();
        totalCreated += s.getTotalCreated();
        totalDestroyed += s.getTotalBadDestroyed();
        totalCreateErrors += s.getTotalCreateErrors();
        totalDestroyErrors += s.getTotalDestroyErrors();
        totalBadDestroyed += s.getTotalBadDestroyed();
        totalTimedOut += s.getTotalTimedOut();
        checkedOut += s.getCheckedOut();
        maxPoolSize += s.getMaxPoolSize();
        minPoolSize += s.getMinPoolSize();
        poolSize += s.getPoolSize();
        sampleMaxCheckedOut += s.getSampleMaxCheckedOut();
        sampleMaxPoolSize += s.getSampleMaxPoolSize();
        idleCount += s.getIdleCount();
        waitTimeHist.add(s.getWaitTime().getHistogram());
        createTimeHist.add(s.getLifecycleStats().getCreateTime().getHistogram());
    }
    _totalCreated = totalCreated;
    _totalDestroyed = totalDestroyed;
    _totalBadDestroyed = totalBadDestroyed;
    _totalCreateErrors = totalCreateErrors;
    _totalDestroyErrors = totalDestroyErrors;
    _totalTimedOut = totalTimedOut;
    _checkedOut = checkedOut;
    _maxPoolSize = maxPoolSize;
    _minPoolSize = minPoolSize;
    _poolSize = poolSize;
    _sampleMaxCheckedOut = sampleMaxCheckedOut;
    _sampleMaxPoolSize = sampleMaxPoolSize;
    _idleCount = idleCount;
    _waitTime = new LatencyMetric(waitTimeHist);
    _lifecycleStats = new LifecycleStats(new LatencyMetric<AggregatedHistogram<Sampling>>(createTimeHist));
}

Example 2 with LatencyMetric

use of com.linkedin.pinot.common.metrics.LatencyMetric in project pinot by linkedin.

the class ScatterGatherPerfTester method run.

public void run() throws Exception {
    List<ScatterGatherPerfServer> servers = null;
    // Run Servers when mode is RUN_SERVER or RUN_BOTH
    if (_mode != ExecutionMode.RUN_CLIENT) {
        servers = runServer();
    }
    if (_mode != ExecutionMode.RUN_SERVER) {
        int port = _startPortNum;
        // Setup Routing config for clients
        RoutingTableConfig config = new RoutingTableConfig();
        Map<String, PerTableRoutingConfig> cfg = config.getPerTableRoutingCfg();
        PerTableRoutingConfig c = new PerTableRoutingConfig(null);
        Map<Integer, List<ServerInstance>> instanceMap = c.getNodeToInstancesMap();
        port = _startPortNum;
        int numUniqueServers = _remoteServerHosts.size();
        for (int i = 0; i < _numServers; i++) {
            List<ServerInstance> instances = new ArrayList<ServerInstance>();
            String server = null;
            if (_mode == ExecutionMode.RUN_BOTH)
                server = "localhost";
            else
                server = _remoteServerHosts.get(i % numUniqueServers);
            ServerInstance instance = new ServerInstance(server, port++);
            instances.add(instance);
            instanceMap.put(i, instances);
        }
        String server = null;
        if (_mode == ExecutionMode.RUN_BOTH)
            server = "localhost";
        else
            server = _remoteServerHosts.get(0);
        c.getDefaultServers().add(new ServerInstance(server, port - 1));
        cfg.put(_resourceName, c);
        System.out.println("Routing Config is :" + cfg);
        // Build Clients
        List<Thread> clientThreads = new ArrayList<Thread>();
        List<ScatterGatherPerfClient> clients = new ArrayList<ScatterGatherPerfClient>();
        AggregatedHistogram<Histogram> latencyHistogram = new AggregatedHistogram<Histogram>();
        for (int i = 0; i < _numClients; i++) {
            ScatterGatherPerfClient c2 = new ScatterGatherPerfClient(config, _requestSize, _resourceName, _asyncRequestDispatch, _numRequests, _maxActiveConnectionsPerClientServerPair, _numResponseReaderThreads);
            Thread t = new Thread(c2);
            clients.add(c2);
            latencyHistogram.add(c2.getLatencyHistogram());
            clientThreads.add(t);
        }
        System.out.println("Starting the clients !!");
        long startTimeMs = 0;
        // Start Clients
        for (Thread t2 : clientThreads) t2.start();
        System.out.println("Waiting for clients to finish");
        // Wait for clients to finish
        for (Thread t2 : clientThreads) t2.join();
        Thread.sleep(3000);
        System.out.println("Client threads done !!");
        int totalRequestsMeasured = 0;
        long beginRequestTime = Long.MAX_VALUE;
        long endResponseTime = Long.MIN_VALUE;
        for (ScatterGatherPerfClient c3 : clients) {
            int numRequestsMeasured = c3.getNumRequestsMeasured();
            totalRequestsMeasured += numRequestsMeasured;
            beginRequestTime = Math.min(beginRequestTime, c3.getBeginFirstRequestTime());
            endResponseTime = Math.max(endResponseTime, c3.getEndLastResponseTime());
        //System.out.println("2 Num Requests :" + numRequestsMeasured);
        //System.out.println("2 time :" + timeTakenMs );
        //System.out.println("2 Throughput (Requests/Second) :" + ((numRequestsMeasured* 1.0 * 1000)/timeTakenMs));
        }
        long totalTimeTakenMs = endResponseTime - beginRequestTime;
        System.out.println("Overall Total Num Requests :" + totalRequestsMeasured);
        System.out.println("Overall Total time :" + totalTimeTakenMs);
        System.out.println("Overall Throughput (Requests/Second) :" + ((totalRequestsMeasured * 1.0 * 1000) / totalTimeTakenMs));
        latencyHistogram.refresh();
        System.out.println("Latency :" + new LatencyMetric<AggregatedHistogram<Histogram>>(latencyHistogram));
    }
    if (_mode == ExecutionMode.RUN_BOTH) {
        // Shutdown Servers
        for (ScatterGatherPerfServer s : servers) {
            s.shutdown();
        }
    }
}

Example 3 with LatencyMetric

use of com.linkedin.pinot.common.metrics.LatencyMetric in project pinot by linkedin.

the class ScatterGatherPerfClient method run.

@Override
public void run() {
    System.out.println("Client starting !!");
    try {
        List<ServerInstance> s1 = new ArrayList<ServerInstance>();
        ServerInstance s = new ServerInstance("localhost", 9099);
        s1.add(s);
        SimpleScatterGatherRequest req = null;
        TimerContext tc = null;
        for (int i = 0; i < _numRequests; i++) {
            LOGGER.debug("Sending request number {}", i);
            do {
                req = getRequest();
            } while ((null == req));
            if (i == _numRequestsToSkipForMeasurement) {
                tc = MetricsHelper.startTimer();
                _beginFirstRequestTime = System.currentTimeMillis();
            }
            if (i >= _numRequestsToSkipForMeasurement) {
                _numRequestsMeasured++;
            }
            final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
            if (!_asyncRequestSubmit) {
                sendRequestAndGetResponse(req, scatterGatherStats);
                _endLastResponseTime = System.currentTimeMillis();
            } else {
                CompositeFuture<ServerInstance, ByteBuf> future = asyncSendRequestAndGetResponse(req, scatterGatherStats);
                _queue.offer(new QueueEntry(false, i >= _numRequestsToSkipForMeasurement, System.currentTimeMillis(), future));
            }
            //System.out.println("Response is :" + r);
            //System.out.println("\n\n");
            req = null;
        }
        if (_asyncRequestSubmit) {
            int numTerminalEntries = _readerThreads.size();
            for (int i = 0; i < numTerminalEntries; i++) {
                _queue.offer(new QueueEntry(true, false, System.currentTimeMillis(), null));
            }
            for (AsyncReader r : _readerThreads) {
                r.join();
            }
        }
        if (null != tc) {
            tc.stop();
            _timerContext = tc;
            System.out.println("Num Requests :" + _numRequestsMeasured);
            System.out.println("Total time :" + tc.getLatencyMs());
            System.out.println("Throughput (Requests/Second) :" + ((_numRequestsMeasured * 1.0 * 1000) / tc.getLatencyMs()));
            System.out.println("Latency :" + new LatencyMetric<Histogram>(_latencyHistogram));
            System.out.println("Scatter-Gather Latency :" + new LatencyMetric<Histogram>(_scatterGather.getLatency()));
        }
    } catch (Exception ex) {
        System.err.println("Client stopped abnormally ");
        ex.printStackTrace();
    }
    shutdown();
    System.out.println("Client done !!");
}