Examples with AirlineDataStream com.linkedin.pinot.tools.streams.AirlineDataStream used on opensource projects

Example 1 with AirlineDataStream

use of com.linkedin.pinot.tools.streams.AirlineDataStream in project pinot by linkedin.

the class HybridQuickstart method execute.

public void execute() throws JSONException, Exception {
    QuickstartTableRequest offlineRequest = prepareOfflineTableRequest();
    QuickstartTableRequest realtimeTableRequest = prepareRealtimeTableRequest();
    File tempDir = new File("/tmp/" + System.currentTimeMillis());
    tempDir.mkdir();
    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(offlineRequest, realtimeTableRequest), 2, 2, 1, tempDir, false);
    printStatus(color.YELLOW, "***** starting kafka  *****");
    startKafka();
    printStatus(color.YELLOW, "***** starting 2 servers, 2 brokers and 1 controller  *****");
    runner.startAll();
    printStatus(color.YELLOW, "***** creating a server tenant with name  airline  *****");
    runner.createServerTenantWith(1, 1, "airline");
    printStatus(color.YELLOW, "***** creating a broker tenant with name  airline_broker  *****");
    runner.createBrokerTenantWith(2, "airline_broker");
    printStatus(color.YELLOW, "***** adding airline schema  *****");
    runner.addSchema();
    printStatus(color.YELLOW, "***** adding airline offline and realtime table  *****");
    runner.addTable();
    printStatus(color.YELLOW, "***** adding airline offline segment  *****");
    runner.buildSegment();
    printStatus(color.YELLOW, "***** pushing airline offline segment to the controller  *****");
    runner.pushSegment();
    File dataFile = new File(_offlineQuickStartDataDir + "/airline.avro");
    File schemaFile = new File(_offlineQuickStartDataDir + "/airlineStats.schema");
    printStatus(color.YELLOW, "***** publishing data to kafka for airline realtime to start consuming event stream  *****");
    final AirlineDataStream stream = new AirlineDataStream(Schema.fromFile(schemaFile), dataFile);
    stream.run();
    printStatus(color.YELLOW, "***** Pinot Hybrid with hybrid table setup is complete *****");
    printStatus(color.YELLOW, "*****    1. Sequence of operations *****");
    printStatus(color.YELLOW, "*****    2. Started 1 controller instances where tenant creation is enabled *****");
    printStatus(color.YELLOW, "*****    3. Started 2 servers and 2 brokers *****");
    printStatus(color.YELLOW, "*****    4. created a server tenant with 1 offline and 1 realtime instance *****");
    printStatus(color.YELLOW, "*****    5. Created a broker tenant with 2 instances *****");
    printStatus(color.YELLOW, "*****    6. Added a schema *****");
    printStatus(color.YELLOW, "*****    7. Created a offline and a realtime table with the tenant names created abover *****");
    printStatus(color.YELLOW, "*****    8. Created and published an offline segment *****");
    printStatus(color.YELLOW, "***** Also Started publishing a kafka stream for the realtime instance so start consuming *****");
    printStatus(color.YELLOW, "***** go to http://localhost:9000/query to run a few queries *****");
    Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            try {
                stream.shutdown();
                Thread.sleep(2000);
                printStatus(color.GREEN, "***** shutting down hybrid quick start *****");
                runner.stop();
                runner.clean();
                FileUtils.deleteDirectory(_offlineQuickStartDataDir);
                FileUtils.deleteDirectory(_realtimeQuickStartDataDir);
                KafkaStarterUtils.stopServer(kafkaStarter);
                ZkStarter.stopLocalZkServer(_zookeeperInstance);
            } catch (Exception e) {
            }
        }
    });
}