Examples with KafkaServerStartable kafka.server.KafkaServerStartable used on opensource projects

Example 1 with KafkaServerStartable

use of kafka.server.KafkaServerStartable in project pinot by linkedin.

the class KafkaStarterUtils method startServer.

public static KafkaServerStartable startServer(final int port, final int brokerId, final String zkStr, final String logDirPath, final Properties configuration) {
    // Create the ZK nodes for Kafka, if needed
    int indexOfFirstSlash = zkStr.indexOf('/');
    if (indexOfFirstSlash != -1) {
        String bareZkUrl = zkStr.substring(0, indexOfFirstSlash);
        String zkNodePath = zkStr.substring(indexOfFirstSlash);
        ZkClient client = new ZkClient(bareZkUrl);
        client.createPersistent(zkNodePath, true);
        client.close();
    }
    File logDir = new File(logDirPath);
    logDir.mkdirs();
    configureKafkaPort(configuration, port);
    configureZkConnectionString(configuration, zkStr);
    configureBrokerId(configuration, brokerId);
    configureKafkaLogDirectory(configuration, logDir);
    configuration.put("zookeeper.session.timeout.ms", "60000");
    KafkaConfig config = new KafkaConfig(configuration);
    KafkaServerStartable serverStartable = new KafkaServerStartable(config);
    serverStartable.startup();
    return serverStartable;
}

Example 2 with KafkaServerStartable

use of kafka.server.KafkaServerStartable in project pinot by linkedin.

the class BenchmarkRealtimeConsumptionSpeed method runBenchmark.

private void runBenchmark() throws Exception {
    // Start ZK and Kafka
    startZk();
    KafkaServerStartable kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration());
    // Create Kafka topic
    KafkaStarterUtils.createTopic(KAFKA_TOPIC, KafkaStarterUtils.DEFAULT_ZK_STR, 10);
    // Unpack data (needed to get the Avro schema)
    TarGzCompressionUtils.unTar(new File(TestUtils.getFileFromResourceUrl(RealtimeClusterIntegrationTest.class.getClassLoader().getResource("On_Time_On_Time_Performance_2014_100k_subset_nonulls.tar.gz"))), _tmpDir);
    _tmpDir.mkdirs();
    final List<File> avroFiles = new ArrayList<File>(SEGMENT_COUNT);
    for (int segmentNumber = 1; segmentNumber <= SEGMENT_COUNT; ++segmentNumber) {
        avroFiles.add(new File(_tmpDir.getPath() + "/On_Time_On_Time_Performance_2014_" + segmentNumber + ".avro"));
    }
    File schemaFile = new File(OfflineClusterIntegrationTest.class.getClassLoader().getResource("On_Time_On_Time_Performance_2014_100k_subset_nonulls.schema").getFile());
    // Start the Pinot cluster
    startController();
    startBroker();
    startServer();
    // Create realtime table
    setUpTable("mytable", "DaysSinceEpoch", "daysSinceEpoch", KafkaStarterUtils.DEFAULT_ZK_STR, KAFKA_TOPIC, schemaFile, avroFiles.get(0));
    // Wait a couple of seconds for all Helix state transitions to happen
    Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
    // Generate ROW_COUNT rows and write them into Kafka
    new Thread() {

        @Override
        public void run() {
            pushRandomAvroIntoKafka(avroFiles.get(0), KafkaStarterUtils.DEFAULT_KAFKA_BROKER, KAFKA_TOPIC, ROW_COUNT, RANDOM);
        }
    }.start();
    // Count how many seconds it takes for select count(*) to match with ROW_COUNT
    long startTime = System.currentTimeMillis();
    int pinotRecordCount = -1;
    long timeAfterTimeout = System.currentTimeMillis() + TIMEOUT_MILLIS;
    do {
        Thread.sleep(500L);
        // Run the query
        try {
            JSONObject response = postQuery("select count(*) from mytable");
            JSONArray aggregationResultsArray = response.getJSONArray("aggregationResults");
            JSONObject firstAggregationResult = aggregationResultsArray.getJSONObject(0);
            String pinotValue = firstAggregationResult.getString("value");
            pinotRecordCount = Integer.parseInt(pinotValue);
        } catch (Exception e) {
            // Ignore
            continue;
        }
        System.out.println("Pinot record count: " + pinotRecordCount);
        if (timeAfterTimeout < System.currentTimeMillis()) {
            throw new RuntimeException("Timeout exceeded!");
        }
    } while (ROW_COUNT != pinotRecordCount);
    long endTime = System.currentTimeMillis();
    System.out.println("Consumed " + ROW_COUNT + " rows in " + (endTime - startTime) / 1000.0 + " seconds");
}

Example 3 with KafkaServerStartable

use of kafka.server.KafkaServerStartable in project pinot by linkedin.

the class RealtimeStressTest method runBenchmark.

private void runBenchmark() throws Exception {
    // Start ZK and Kafka
    startZk();
    KafkaServerStartable kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration());
    // Create Kafka topic
    KafkaStarterUtils.createTopic(KAFKA_TOPIC, KafkaStarterUtils.DEFAULT_ZK_STR, 10);
    // Unpack data (needed to get the Avro schema)
    TarGzCompressionUtils.unTar(new File(TestUtils.getFileFromResourceUrl(RealtimeClusterIntegrationTest.class.getClassLoader().getResource("On_Time_On_Time_Performance_2014_100k_subset_nonulls.tar.gz"))), _tmpDir);
    _tmpDir.mkdirs();
    final List<File> avroFiles = new ArrayList<File>(SEGMENT_COUNT);
    for (int segmentNumber = 1; segmentNumber <= SEGMENT_COUNT; ++segmentNumber) {
        avroFiles.add(new File(_tmpDir.getPath() + "/On_Time_On_Time_Performance_2014_" + segmentNumber + ".avro"));
    }
    File schemaFile = new File(OfflineClusterIntegrationTest.class.getClassLoader().getResource("On_Time_On_Time_Performance_2014_100k_subset_nonulls.schema").getFile());
    // Start the Pinot cluster
    startController();
    startBroker();
    startServer();
    // Create realtime table
    setUpTable("mytable", "DaysSinceEpoch", "daysSinceEpoch", KafkaStarterUtils.DEFAULT_ZK_STR, KAFKA_TOPIC, schemaFile, avroFiles.get(0));
    // Wait a couple of seconds for all Helix state transitions to happen
    Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
    // Generate ROW_COUNT rows and write them into Kafka
    pushRandomAvroIntoKafka(avroFiles.get(0), KafkaStarterUtils.DEFAULT_KAFKA_BROKER, KAFKA_TOPIC, ROW_COUNT, RANDOM);
    rowsWritten += ROW_COUNT;
    // Run forever until something breaks or the timeout completes
    long pinotRecordCount = -1L;
    long timeAfterTimeout = System.currentTimeMillis() + TIMEOUT_MILLIS;
    do {
        Thread.sleep(500L);
        // Run the query
        try {
            JSONObject response = postQuery("select count(*) from mytable");
            JSONArray aggregationResultsArray = response.getJSONArray("aggregationResults");
            JSONObject firstAggregationResult = aggregationResultsArray.getJSONObject(0);
            String pinotValue = firstAggregationResult.getString("value");
            pinotRecordCount = Long.parseLong(pinotValue);
        } catch (Exception e) {
            // Ignore
            continue;
        }
        // Write more rows if needed
        if (rowsWritten - pinotRecordCount < MIN_ROW_COUNT) {
            pushRandomAvroIntoKafka(avroFiles.get(0), KafkaStarterUtils.DEFAULT_KAFKA_BROKER, KAFKA_TOPIC, ROW_COUNT, RANDOM);
            rowsWritten += ROW_COUNT;
        }
        System.out.println("Pinot record count: " + pinotRecordCount);
        if (timeAfterTimeout < System.currentTimeMillis()) {
            throw new RuntimeException("Timeout exceeded!");
        }
    } while (true);
}

Example 4 with KafkaServerStartable

use of kafka.server.KafkaServerStartable in project pinot by linkedin.

the class RealtimeClusterIntegrationTest method tearDown.

@AfterClass
public void tearDown() throws Exception {
    stopBroker();
    stopController();
    stopServer();
    for (KafkaServerStartable kafkaStarter : kafkaStarters) {
        KafkaStarterUtils.stopServer(kafkaStarter);
    }
    try {
        stopZk();
    } catch (Exception e) {
    // Swallow ZK Exceptions.
    }
    FileUtils.deleteDirectory(_tmpDir);
}

Example 5 with KafkaServerStartable

use of kafka.server.KafkaServerStartable in project pinot by linkedin.

the class RealtimeQuickStart method execute.

public void execute() throws JSONException, Exception {
    _quickStartDataDir = new File("quickStartData" + System.currentTimeMillis());
    String quickStartDataDirName = _quickStartDataDir.getName();
    if (!_quickStartDataDir.exists()) {
        _quickStartDataDir.mkdir();
    }
    File schema = new File(quickStartDataDirName + "/rsvp_pinot_schema.json");
    File tableCreate = new File(quickStartDataDirName + "/rsvp_create_table_request.json");
    FileUtils.copyURLToFile(RealtimeQuickStart.class.getClassLoader().getResource("sample_data/rsvp_pinot_schema.json"), schema);
    FileUtils.copyURLToFile(RealtimeQuickStart.class.getClassLoader().getResource("sample_data/rsvp_create_table_request.json"), tableCreate);
    printStatus(color.CYAN, "Starting Kafka");
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    final KafkaServerStartable kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration());
    KafkaStarterUtils.createTopic("meetupRSVPEvents", KafkaStarterUtils.DEFAULT_ZK_STR, 10);
    File tempDir = new File("/tmp/" + String.valueOf(System.currentTimeMillis()));
    QuickstartTableRequest request = new QuickstartTableRequest("meetupRsvp", schema, tableCreate);
    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, tempDir);
    runner.startAll();
    printStatus(color.CYAN, "Starting controller, server and broker");
    runner.addSchema();
    runner.addTable();
    printStatus(color.CYAN, "Added schema and table");
    printStatus(color.YELLOW, "Realtime quickstart setup complete");
    final MeetupRsvpStream meetupRSVPProvider = new MeetupRsvpStream(schema);
    meetupRSVPProvider.run();
    printStatus(color.CYAN, "Starting meetup data stream and publishing to kafka");
    // lets wait for a few events to get populated
    Thread.sleep(5000);
    String q1 = "select count(*) from meetupRsvp limit 0";
    printStatus(color.YELLOW, "Total number of documents in the table");
    printStatus(color.CYAN, "Query : " + q1);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q1)));
    printStatus(color.GREEN, "***************************************************");
    String q2 = "select sum(rsvp_count) from meetupRsvp group by group_city top 10 limit 0";
    printStatus(color.YELLOW, "Top 10 cities with the most rsvp");
    printStatus(color.CYAN, "Query : " + q2);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q2)));
    printStatus(color.GREEN, "***************************************************");
    String q3 = "select * from meetupRsvp order by mtime limit 10";
    printStatus(color.YELLOW, "Show 10 most recent rsvps");
    printStatus(color.CYAN, "Query : " + q3);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q3)));
    printStatus(color.GREEN, "***************************************************");
    String q4 = "select sum(rsvp_count) from meetupRsvp group by event_name top 10 limit 0";
    printStatus(color.YELLOW, "Show top 10 rsvp'ed events");
    printStatus(color.CYAN, "Query : " + q4);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q4)));
    printStatus(color.GREEN, "***************************************************");
    String q5 = "select count(*) from meetupRsvp limit 0";
    printStatus(color.YELLOW, "Total number of documents in the table");
    printStatus(color.CYAN, "Query : " + q5);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q5)));
    printStatus(color.GREEN, "***************************************************");
    printStatus(color.GREEN, "you can always go to http://localhost:9000/query/ to play around in the query console");
    Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            try {
                printStatus(color.GREEN, "***** shutting down realtime quickstart *****");
                meetupRSVPProvider.stopPublishing();
                FileUtils.deleteDirectory(_quickStartDataDir);
                runner.stop();
                runner.clean();
                KafkaStarterUtils.stopServer(kafkaStarter);
                ZkStarter.stopLocalZkServer(_zookeeperInstance);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    });
    long st = System.currentTimeMillis();
    while (true) {
        if (System.currentTimeMillis() - st >= (60 * 60) * 1000) {
            break;
        }
    }
    printStatus(color.YELLOW, "running since an hour, stopping now");
}