Examples with QuickstartRunner com.linkedin.pinot.tools.admin.command.QuickstartRunner used on opensource projects

Example 1 with QuickstartRunner

use of com.linkedin.pinot.tools.admin.command.QuickstartRunner in project pinot by linkedin.

the class Quickstart method execute.

public boolean execute() throws Exception {
    _quickStartDataDir = new File("quickStartData" + System.currentTimeMillis());
    String quickStartDataDirName = _quickStartDataDir.getName();
    if (!_quickStartDataDir.exists()) {
        _quickStartDataDir.mkdir();
    }
    File schemaFile = new File(quickStartDataDirName + "/baseball.schema");
    File dataFile = new File(quickStartDataDirName + "/baseball.csv");
    File tableCreationJsonFileName = new File(quickStartDataDirName + "/baseballTable.json");
    FileUtils.copyURLToFile(Quickstart.class.getClassLoader().getResource("sample_data/baseball.schema"), schemaFile);
    FileUtils.copyURLToFile(Quickstart.class.getClassLoader().getResource("sample_data/baseball.csv"), dataFile);
    FileUtils.copyURLToFile(Quickstart.class.getClassLoader().getResource("sample_data/baseballTable.json"), tableCreationJsonFileName);
    File tempDirOne = new File("/tmp/" + System.currentTimeMillis());
    tempDirOne.mkdir();
    File tempDir = new File("/tmp/" + String.valueOf(System.currentTimeMillis()));
    String tableName = "baseballStats";
    QuickstartTableRequest request = new QuickstartTableRequest(tableName, schemaFile, tableCreationJsonFileName, _quickStartDataDir, FileFormat.CSV);
    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, tempDir);
    runner.clean();
    runner.startAll();
    printStatus(color.CYAN, "Deployed Zookeeper");
    printStatus(color.CYAN, "Deployed controller, broker and server");
    runner.addSchema();
    printStatus(color.CYAN, "Added baseballStats schema");
    runner.addTable();
    printStatus(color.CYAN, "Creating baseballStats table");
    runner.buildSegment();
    printStatus(color.CYAN, "Built index segment for baseballStats");
    runner.pushSegment();
    printStatus(color.CYAN, "Pushing segments to the controller");
    printStatus(color.CYAN, "Waiting for a second for the server to fetch the assigned segment");
    Thread.sleep(5000);
    Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            try {
                printStatus(color.GREEN, "***** shutting down offline quick start *****");
                FileUtils.deleteDirectory(_quickStartDataDir);
                runner.clean();
                runner.stop();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    });
    printStatus(color.YELLOW, "Offline quickstart complete");
    String q1 = "select count(*) from baseballStats limit 0";
    printStatus(color.YELLOW, "Total number of documents in the table");
    printStatus(color.CYAN, "Query : " + q1);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q1)));
    printStatus(color.GREEN, "***************************************************");
    String q2 = "select sum('runs') from baseballStats group by playerName top 5 limit 0";
    printStatus(color.YELLOW, "Top 5 run scorers of all time ");
    printStatus(color.CYAN, "Query : " + q2);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q2)));
    printStatus(color.GREEN, "***************************************************");
    String q3 = "select sum('runs') from baseballStats where yearID=2000 group by playerName top 5 limit 0";
    printStatus(color.YELLOW, "Top 5 run scorers of the year 2000");
    printStatus(color.CYAN, "Query : " + q3);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q3)));
    printStatus(color.GREEN, "***************************************************");
    String q4 = "select sum('runs') from baseballStats where yearID>=2000 group by playerName limit 0";
    printStatus(color.YELLOW, "Top 10 run scorers after 2000");
    printStatus(color.CYAN, "Query : " + q4);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q4)));
    printStatus(color.GREEN, "***************************************************");
    String q5 = "select playerName,runs,homeRuns from baseballStats order by yearID limit 10";
    printStatus(color.YELLOW, "Print playerName,runs,homeRuns for 10 records from the table and order them by yearID");
    printStatus(color.CYAN, "Query : " + q5);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q5)));
    printStatus(color.GREEN, "***************************************************");
    printStatus(color.GREEN, "you can always go to http://localhost:9000/query/ to play around in the query console");
    long st = System.currentTimeMillis();
    while (true) {
        if (System.currentTimeMillis() - st >= (60 * 60) * 1000) {
            break;
        }
    }
    printStatus(color.YELLOW, "running since an hour, stopping now");
    return true;
}

Example 2 with QuickstartRunner

use of com.linkedin.pinot.tools.admin.command.QuickstartRunner in project pinot by linkedin.

the class RealtimeQuickStart method execute.

public void execute() throws JSONException, Exception {
    _quickStartDataDir = new File("quickStartData" + System.currentTimeMillis());
    String quickStartDataDirName = _quickStartDataDir.getName();
    if (!_quickStartDataDir.exists()) {
        _quickStartDataDir.mkdir();
    }
    File schema = new File(quickStartDataDirName + "/rsvp_pinot_schema.json");
    File tableCreate = new File(quickStartDataDirName + "/rsvp_create_table_request.json");
    FileUtils.copyURLToFile(RealtimeQuickStart.class.getClassLoader().getResource("sample_data/rsvp_pinot_schema.json"), schema);
    FileUtils.copyURLToFile(RealtimeQuickStart.class.getClassLoader().getResource("sample_data/rsvp_create_table_request.json"), tableCreate);
    printStatus(color.CYAN, "Starting Kafka");
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    final KafkaServerStartable kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration());
    KafkaStarterUtils.createTopic("meetupRSVPEvents", KafkaStarterUtils.DEFAULT_ZK_STR, 10);
    File tempDir = new File("/tmp/" + String.valueOf(System.currentTimeMillis()));
    QuickstartTableRequest request = new QuickstartTableRequest("meetupRsvp", schema, tableCreate);
    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, tempDir);
    runner.startAll();
    printStatus(color.CYAN, "Starting controller, server and broker");
    runner.addSchema();
    runner.addTable();
    printStatus(color.CYAN, "Added schema and table");
    printStatus(color.YELLOW, "Realtime quickstart setup complete");
    final MeetupRsvpStream meetupRSVPProvider = new MeetupRsvpStream(schema);
    meetupRSVPProvider.run();
    printStatus(color.CYAN, "Starting meetup data stream and publishing to kafka");
    // lets wait for a few events to get populated
    Thread.sleep(5000);
    String q1 = "select count(*) from meetupRsvp limit 0";
    printStatus(color.YELLOW, "Total number of documents in the table");
    printStatus(color.CYAN, "Query : " + q1);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q1)));
    printStatus(color.GREEN, "***************************************************");
    String q2 = "select sum(rsvp_count) from meetupRsvp group by group_city top 10 limit 0";
    printStatus(color.YELLOW, "Top 10 cities with the most rsvp");
    printStatus(color.CYAN, "Query : " + q2);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q2)));
    printStatus(color.GREEN, "***************************************************");
    String q3 = "select * from meetupRsvp order by mtime limit 10";
    printStatus(color.YELLOW, "Show 10 most recent rsvps");
    printStatus(color.CYAN, "Query : " + q3);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q3)));
    printStatus(color.GREEN, "***************************************************");
    String q4 = "select sum(rsvp_count) from meetupRsvp group by event_name top 10 limit 0";
    printStatus(color.YELLOW, "Show top 10 rsvp'ed events");
    printStatus(color.CYAN, "Query : " + q4);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q4)));
    printStatus(color.GREEN, "***************************************************");
    String q5 = "select count(*) from meetupRsvp limit 0";
    printStatus(color.YELLOW, "Total number of documents in the table");
    printStatus(color.CYAN, "Query : " + q5);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q5)));
    printStatus(color.GREEN, "***************************************************");
    printStatus(color.GREEN, "you can always go to http://localhost:9000/query/ to play around in the query console");
    Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            try {
                printStatus(color.GREEN, "***** shutting down realtime quickstart *****");
                meetupRSVPProvider.stopPublishing();
                FileUtils.deleteDirectory(_quickStartDataDir);
                runner.stop();
                runner.clean();
                KafkaStarterUtils.stopServer(kafkaStarter);
                ZkStarter.stopLocalZkServer(_zookeeperInstance);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    });
    long st = System.currentTimeMillis();
    while (true) {
        if (System.currentTimeMillis() - st >= (60 * 60) * 1000) {
            break;
        }
    }
    printStatus(color.YELLOW, "running since an hour, stopping now");
}

Example 3 with QuickstartRunner

use of com.linkedin.pinot.tools.admin.command.QuickstartRunner in project pinot by linkedin.

the class HybridQuickstart method execute.

public void execute() throws JSONException, Exception {
    QuickstartTableRequest offlineRequest = prepareOfflineTableRequest();
    QuickstartTableRequest realtimeTableRequest = prepareRealtimeTableRequest();
    File tempDir = new File("/tmp/" + System.currentTimeMillis());
    tempDir.mkdir();
    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(offlineRequest, realtimeTableRequest), 2, 2, 1, tempDir, false);
    printStatus(color.YELLOW, "***** starting kafka  *****");
    startKafka();
    printStatus(color.YELLOW, "***** starting 2 servers, 2 brokers and 1 controller  *****");
    runner.startAll();
    printStatus(color.YELLOW, "***** creating a server tenant with name  airline  *****");
    runner.createServerTenantWith(1, 1, "airline");
    printStatus(color.YELLOW, "***** creating a broker tenant with name  airline_broker  *****");
    runner.createBrokerTenantWith(2, "airline_broker");
    printStatus(color.YELLOW, "***** adding airline schema  *****");
    runner.addSchema();
    printStatus(color.YELLOW, "***** adding airline offline and realtime table  *****");
    runner.addTable();
    printStatus(color.YELLOW, "***** adding airline offline segment  *****");
    runner.buildSegment();
    printStatus(color.YELLOW, "***** pushing airline offline segment to the controller  *****");
    runner.pushSegment();
    File dataFile = new File(_offlineQuickStartDataDir + "/airline.avro");
    File schemaFile = new File(_offlineQuickStartDataDir + "/airlineStats.schema");
    printStatus(color.YELLOW, "***** publishing data to kafka for airline realtime to start consuming event stream  *****");
    final AirlineDataStream stream = new AirlineDataStream(Schema.fromFile(schemaFile), dataFile);
    stream.run();
    printStatus(color.YELLOW, "***** Pinot Hybrid with hybrid table setup is complete *****");
    printStatus(color.YELLOW, "*****    1. Sequence of operations *****");
    printStatus(color.YELLOW, "*****    2. Started 1 controller instances where tenant creation is enabled *****");
    printStatus(color.YELLOW, "*****    3. Started 2 servers and 2 brokers *****");
    printStatus(color.YELLOW, "*****    4. created a server tenant with 1 offline and 1 realtime instance *****");
    printStatus(color.YELLOW, "*****    5. Created a broker tenant with 2 instances *****");
    printStatus(color.YELLOW, "*****    6. Added a schema *****");
    printStatus(color.YELLOW, "*****    7. Created a offline and a realtime table with the tenant names created abover *****");
    printStatus(color.YELLOW, "*****    8. Created and published an offline segment *****");
    printStatus(color.YELLOW, "***** Also Started publishing a kafka stream for the realtime instance so start consuming *****");
    printStatus(color.YELLOW, "***** go to http://localhost:9000/query to run a few queries *****");
    Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            try {
                stream.shutdown();
                Thread.sleep(2000);
                printStatus(color.GREEN, "***** shutting down hybrid quick start *****");
                runner.stop();
                runner.clean();
                FileUtils.deleteDirectory(_offlineQuickStartDataDir);
                FileUtils.deleteDirectory(_realtimeQuickStartDataDir);
                KafkaStarterUtils.stopServer(kafkaStarter);
                ZkStarter.stopLocalZkServer(_zookeeperInstance);
            } catch (Exception e) {
            }
        }
    });
}