Examples with AggregatedHistogram com.linkedin.pinot.common.metrics.AggregatedHistogram used on opensource projects

Example 1 with AggregatedHistogram

use of com.linkedin.pinot.common.metrics.AggregatedHistogram in project pinot by linkedin.

the class AggregatedPoolStats method refresh.

public void refresh() {
    int totalCreated = 0;
    int totalDestroyed = 0;
    int totalCreateErrors = 0;
    int totalDestroyErrors = 0;
    int totalBadDestroyed = 0;
    int totalTimedOut = 0;
    int checkedOut = 0;
    int maxPoolSize = 0;
    int minPoolSize = 0;
    int poolSize = 0;
    int sampleMaxCheckedOut = 0;
    int sampleMaxPoolSize = 0;
    int idleCount = 0;
    AggregatedHistogram<Sampling> waitTimeHist = new AggregatedHistogram<Sampling>();
    AggregatedHistogram<Sampling> createTimeHist = new AggregatedHistogram<Sampling>();
    for (PoolStatsProvider p : _poolStatsProvider) {
        PoolStats<T> s = p.getStats();
        totalCreated += s.getTotalCreated();
        totalDestroyed += s.getTotalBadDestroyed();
        totalCreateErrors += s.getTotalCreateErrors();
        totalDestroyErrors += s.getTotalDestroyErrors();
        totalBadDestroyed += s.getTotalBadDestroyed();
        totalTimedOut += s.getTotalTimedOut();
        checkedOut += s.getCheckedOut();
        maxPoolSize += s.getMaxPoolSize();
        minPoolSize += s.getMinPoolSize();
        poolSize += s.getPoolSize();
        sampleMaxCheckedOut += s.getSampleMaxCheckedOut();
        sampleMaxPoolSize += s.getSampleMaxPoolSize();
        idleCount += s.getIdleCount();
        waitTimeHist.add(s.getWaitTime().getHistogram());
        createTimeHist.add(s.getLifecycleStats().getCreateTime().getHistogram());
    }
    _totalCreated = totalCreated;
    _totalDestroyed = totalDestroyed;
    _totalBadDestroyed = totalBadDestroyed;
    _totalCreateErrors = totalCreateErrors;
    _totalDestroyErrors = totalDestroyErrors;
    _totalTimedOut = totalTimedOut;
    _checkedOut = checkedOut;
    _maxPoolSize = maxPoolSize;
    _minPoolSize = minPoolSize;
    _poolSize = poolSize;
    _sampleMaxCheckedOut = sampleMaxCheckedOut;
    _sampleMaxPoolSize = sampleMaxPoolSize;
    _idleCount = idleCount;
    _waitTime = new LatencyMetric(waitTimeHist);
    _lifecycleStats = new LifecycleStats(new LatencyMetric<AggregatedHistogram<Sampling>>(createTimeHist));
}

Example 2 with AggregatedHistogram

use of com.linkedin.pinot.common.metrics.AggregatedHistogram in project pinot by linkedin.

the class ScatterGatherPerfTester method run.

public void run() throws Exception {
    List<ScatterGatherPerfServer> servers = null;
    // Run Servers when mode is RUN_SERVER or RUN_BOTH
    if (_mode != ExecutionMode.RUN_CLIENT) {
        servers = runServer();
    }
    if (_mode != ExecutionMode.RUN_SERVER) {
        int port = _startPortNum;
        // Setup Routing config for clients
        RoutingTableConfig config = new RoutingTableConfig();
        Map<String, PerTableRoutingConfig> cfg = config.getPerTableRoutingCfg();
        PerTableRoutingConfig c = new PerTableRoutingConfig(null);
        Map<Integer, List<ServerInstance>> instanceMap = c.getNodeToInstancesMap();
        port = _startPortNum;
        int numUniqueServers = _remoteServerHosts.size();
        for (int i = 0; i < _numServers; i++) {
            List<ServerInstance> instances = new ArrayList<ServerInstance>();
            String server = null;
            if (_mode == ExecutionMode.RUN_BOTH)
                server = "localhost";
            else
                server = _remoteServerHosts.get(i % numUniqueServers);
            ServerInstance instance = new ServerInstance(server, port++);
            instances.add(instance);
            instanceMap.put(i, instances);
        }
        String server = null;
        if (_mode == ExecutionMode.RUN_BOTH)
            server = "localhost";
        else
            server = _remoteServerHosts.get(0);
        c.getDefaultServers().add(new ServerInstance(server, port - 1));
        cfg.put(_resourceName, c);
        System.out.println("Routing Config is :" + cfg);
        // Build Clients
        List<Thread> clientThreads = new ArrayList<Thread>();
        List<ScatterGatherPerfClient> clients = new ArrayList<ScatterGatherPerfClient>();
        AggregatedHistogram<Histogram> latencyHistogram = new AggregatedHistogram<Histogram>();
        for (int i = 0; i < _numClients; i++) {
            ScatterGatherPerfClient c2 = new ScatterGatherPerfClient(config, _requestSize, _resourceName, _asyncRequestDispatch, _numRequests, _maxActiveConnectionsPerClientServerPair, _numResponseReaderThreads);
            Thread t = new Thread(c2);
            clients.add(c2);
            latencyHistogram.add(c2.getLatencyHistogram());
            clientThreads.add(t);
        }
        System.out.println("Starting the clients !!");
        long startTimeMs = 0;
        // Start Clients
        for (Thread t2 : clientThreads) t2.start();
        System.out.println("Waiting for clients to finish");
        // Wait for clients to finish
        for (Thread t2 : clientThreads) t2.join();
        Thread.sleep(3000);
        System.out.println("Client threads done !!");
        int totalRequestsMeasured = 0;
        long beginRequestTime = Long.MAX_VALUE;
        long endResponseTime = Long.MIN_VALUE;
        for (ScatterGatherPerfClient c3 : clients) {
            int numRequestsMeasured = c3.getNumRequestsMeasured();
            totalRequestsMeasured += numRequestsMeasured;
            beginRequestTime = Math.min(beginRequestTime, c3.getBeginFirstRequestTime());
            endResponseTime = Math.max(endResponseTime, c3.getEndLastResponseTime());
        //System.out.println("2 Num Requests :" + numRequestsMeasured);
        //System.out.println("2 time :" + timeTakenMs );
        //System.out.println("2 Throughput (Requests/Second) :" + ((numRequestsMeasured* 1.0 * 1000)/timeTakenMs));
        }
        long totalTimeTakenMs = endResponseTime - beginRequestTime;
        System.out.println("Overall Total Num Requests :" + totalRequestsMeasured);
        System.out.println("Overall Total time :" + totalTimeTakenMs);
        System.out.println("Overall Throughput (Requests/Second) :" + ((totalRequestsMeasured * 1.0 * 1000) / totalTimeTakenMs));
        latencyHistogram.refresh();
        System.out.println("Latency :" + new LatencyMetric<AggregatedHistogram<Histogram>>(latencyHistogram));
    }
    if (_mode == ExecutionMode.RUN_BOTH) {
        // Shutdown Servers
        for (ScatterGatherPerfServer s : servers) {
            s.shutdown();
        }
    }
}