Examples with PerfBenchmarkDriver com.linkedin.pinot.tools.perf.PerfBenchmarkDriver used on opensource projects

Example 1 with PerfBenchmarkDriver

use of com.linkedin.pinot.tools.perf.PerfBenchmarkDriver in project pinot by linkedin.

the class BenchmarkQueryEngine method startPinot.

@Setup
public void startPinot() throws Exception {
    System.out.println("Using table name " + TABLE_NAME);
    System.out.println("Using data directory " + DATA_DIRECTORY);
    System.out.println("Starting pinot");
    PerfBenchmarkDriverConf conf = new PerfBenchmarkDriverConf();
    conf.setStartBroker(true);
    conf.setStartController(true);
    conf.setStartServer(true);
    conf.setStartZookeeper(true);
    conf.setUploadIndexes(false);
    conf.setRunQueries(false);
    conf.setServerInstanceSegmentTarDir(null);
    conf.setServerInstanceDataDir(DATA_DIRECTORY);
    conf.setConfigureResources(false);
    _perfBenchmarkDriver = new PerfBenchmarkDriver(conf);
    _perfBenchmarkDriver.run();
    Set<String> tables = new HashSet<String>();
    File[] segments = new File(DATA_DIRECTORY, TABLE_NAME).listFiles();
    for (File segmentDir : segments) {
        SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(segmentDir);
        if (!tables.contains(segmentMetadata.getTableName())) {
            _perfBenchmarkDriver.configureTable(segmentMetadata.getTableName());
            tables.add(segmentMetadata.getTableName());
        }
        System.out.println("Adding segment " + segmentDir.getAbsolutePath());
        _perfBenchmarkDriver.addSegment(segmentMetadata);
    }
    ZkClient client = new ZkClient("localhost:2191", 10000, 10000, new ZNRecordSerializer());
    ZNRecord record = client.readData("/PinotPerfTestCluster/EXTERNALVIEW/" + TABLE_NAME);
    while (true) {
        System.out.println("record = " + record);
        Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS);
        int onlineSegmentCount = 0;
        for (Map<String, String> instancesAndStates : record.getMapFields().values()) {
            for (String state : instancesAndStates.values()) {
                if (state.equals("ONLINE")) {
                    onlineSegmentCount++;
                    break;
                }
            }
        }
        System.out.println(onlineSegmentCount + " segments online out of " + segments.length);
        if (onlineSegmentCount == segments.length) {
            break;
        }
        record = client.readData("/PinotPerfTestCluster/EXTERNALVIEW/" + TABLE_NAME);
    }
    ranOnce = false;
    System.out.println(_perfBenchmarkDriver.postQuery(QUERY_PATTERNS[queryPattern], optimizationFlags).toString(2));
}

Example 2 with PerfBenchmarkDriver

use of com.linkedin.pinot.tools.perf.PerfBenchmarkDriver in project pinot by linkedin.

the class PerfBenchmarkRunner method startServerWithPreLoadedSegments.

/**
   * Start Pinot server with pre-loaded segments.
   *
   * @param dataDir data directory.
   * @param tableNames list of table names to be loaded.
   * @param invertedIndexColumns list of inverted index columns.
   * @throws Exception
   */
public static void startServerWithPreLoadedSegments(String dataDir, List<String> tableNames, List<String> invertedIndexColumns) throws Exception {
    LOGGER.info("Starting server and uploading segments.");
    PerfBenchmarkDriver driver = PerfBenchmarkDriver.startComponents(false, false, false, true, dataDir);
    for (String tableName : tableNames) {
        com.linkedin.pinot.tools.perf.PerfBenchmarkRunner.loadTable(driver, dataDir, tableName, invertedIndexColumns);
    }
}