Examples with RoutingTableConfig com.linkedin.pinot.transport.config.RoutingTableConfig used on opensource projects

Example 1 with RoutingTableConfig

use of com.linkedin.pinot.transport.config.RoutingTableConfig in project pinot by linkedin.

the class ScatterGatherPerfClient method main.

public static void main(String[] args) throws Exception {
    //Process Command Line to get config and port
    CommandLineParser cliParser = new GnuParser();
    Options cliOptions = buildCommandLineOptions();
    CommandLine cmd = cliParser.parse(cliOptions, args, true);
    if ((!cmd.hasOption(BROKER_CONFIG_OPT_NAME)) || (!cmd.hasOption(REQUEST_SIZE_OPT_NAME)) || (!cmd.hasOption(TABLE_NAME_OPT_NAME)) || (!cmd.hasOption(TABLE_NAME_OPT_NAME))) {
        System.err.println("Missing required arguments !!");
        System.err.println(cliOptions);
        throw new RuntimeException("Missing required arguments !!");
    }
    String brokerConfigPath = cmd.getOptionValue(BROKER_CONFIG_OPT_NAME);
    int requestSize = Integer.parseInt(cmd.getOptionValue(REQUEST_SIZE_OPT_NAME));
    int numRequests = Integer.parseInt(cmd.getOptionValue(NUM_REQUESTS_OPT_NAME));
    String resourceName = cmd.getOptionValue(TABLE_NAME_OPT_NAME);
    // build  brokerConf
    PropertiesConfiguration brokerConf = new PropertiesConfiguration();
    brokerConf.setDelimiterParsingDisabled(false);
    brokerConf.load(brokerConfigPath);
    RoutingTableConfig config = new RoutingTableConfig();
    config.init(brokerConf.subset(ROUTING_CFG_PREFIX));
    ScatterGatherPerfClient client = new ScatterGatherPerfClient(config, requestSize, resourceName, false, numRequests, 1, 1);
    client.run();
    System.out.println("Shutting down !!");
    client.shutdown();
    System.out.println("Shut down complete !!");
}

Example 2 with RoutingTableConfig

use of com.linkedin.pinot.transport.config.RoutingTableConfig in project pinot by linkedin.

the class ScatterGatherPerfTester method run.

public void run() throws Exception {
    List<ScatterGatherPerfServer> servers = null;
    // Run Servers when mode is RUN_SERVER or RUN_BOTH
    if (_mode != ExecutionMode.RUN_CLIENT) {
        servers = runServer();
    }
    if (_mode != ExecutionMode.RUN_SERVER) {
        int port = _startPortNum;
        // Setup Routing config for clients
        RoutingTableConfig config = new RoutingTableConfig();
        Map<String, PerTableRoutingConfig> cfg = config.getPerTableRoutingCfg();
        PerTableRoutingConfig c = new PerTableRoutingConfig(null);
        Map<Integer, List<ServerInstance>> instanceMap = c.getNodeToInstancesMap();
        port = _startPortNum;
        int numUniqueServers = _remoteServerHosts.size();
        for (int i = 0; i < _numServers; i++) {
            List<ServerInstance> instances = new ArrayList<ServerInstance>();
            String server = null;
            if (_mode == ExecutionMode.RUN_BOTH)
                server = "localhost";
            else
                server = _remoteServerHosts.get(i % numUniqueServers);
            ServerInstance instance = new ServerInstance(server, port++);
            instances.add(instance);
            instanceMap.put(i, instances);
        }
        String server = null;
        if (_mode == ExecutionMode.RUN_BOTH)
            server = "localhost";
        else
            server = _remoteServerHosts.get(0);
        c.getDefaultServers().add(new ServerInstance(server, port - 1));
        cfg.put(_resourceName, c);
        System.out.println("Routing Config is :" + cfg);
        // Build Clients
        List<Thread> clientThreads = new ArrayList<Thread>();
        List<ScatterGatherPerfClient> clients = new ArrayList<ScatterGatherPerfClient>();
        AggregatedHistogram<Histogram> latencyHistogram = new AggregatedHistogram<Histogram>();
        for (int i = 0; i < _numClients; i++) {
            ScatterGatherPerfClient c2 = new ScatterGatherPerfClient(config, _requestSize, _resourceName, _asyncRequestDispatch, _numRequests, _maxActiveConnectionsPerClientServerPair, _numResponseReaderThreads);
            Thread t = new Thread(c2);
            clients.add(c2);
            latencyHistogram.add(c2.getLatencyHistogram());
            clientThreads.add(t);
        }
        System.out.println("Starting the clients !!");
        long startTimeMs = 0;
        // Start Clients
        for (Thread t2 : clientThreads) t2.start();
        System.out.println("Waiting for clients to finish");
        // Wait for clients to finish
        for (Thread t2 : clientThreads) t2.join();
        Thread.sleep(3000);
        System.out.println("Client threads done !!");
        int totalRequestsMeasured = 0;
        long beginRequestTime = Long.MAX_VALUE;
        long endResponseTime = Long.MIN_VALUE;
        for (ScatterGatherPerfClient c3 : clients) {
            int numRequestsMeasured = c3.getNumRequestsMeasured();
            totalRequestsMeasured += numRequestsMeasured;
            beginRequestTime = Math.min(beginRequestTime, c3.getBeginFirstRequestTime());
            endResponseTime = Math.max(endResponseTime, c3.getEndLastResponseTime());
        //System.out.println("2 Num Requests :" + numRequestsMeasured);
        //System.out.println("2 time :" + timeTakenMs );
        //System.out.println("2 Throughput (Requests/Second) :" + ((numRequestsMeasured* 1.0 * 1000)/timeTakenMs));
        }
        long totalTimeTakenMs = endResponseTime - beginRequestTime;
        System.out.println("Overall Total Num Requests :" + totalRequestsMeasured);
        System.out.println("Overall Total time :" + totalTimeTakenMs);
        System.out.println("Overall Throughput (Requests/Second) :" + ((totalRequestsMeasured * 1.0 * 1000) / totalTimeTakenMs));
        latencyHistogram.refresh();
        System.out.println("Latency :" + new LatencyMetric<AggregatedHistogram<Histogram>>(latencyHistogram));
    }
    if (_mode == ExecutionMode.RUN_BOTH) {
        // Shutdown Servers
        for (ScatterGatherPerfServer s : servers) {
            s.shutdown();
        }
    }
}