Examples with TweetMessageIterator org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessageIterator used on opensource projects

Example 1 with TweetMessageIterator

use of org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessageIterator in project asterixdb by apache.

the class SyntheticDataGeneratorForSpatialIndexEvaluation method main.

public static void main(String[] args) throws Exception {
    SyntheticDataGeneratorConfig config = new SyntheticDataGeneratorConfig();
    CmdLineParser clp = new CmdLineParser(config);
    try {
        clp.parseArgument(args);
    } catch (CmdLineException e) {
        System.err.println(e.getMessage());
        clp.printUsage(System.err);
        System.exit(1);
    }
    //prepare data generator
    GULongIDGenerator uidGenerator = new GULongIDGenerator(config.getPartitionId(), (byte) (0));
    String pointSourceFilePath = config.getPointSourceFile();
    int pointSampleInterval = config.getpointSamplingInterval();
    DataGeneratorForSpatialIndexEvaluation dataGenerator = new DataGeneratorForSpatialIndexEvaluation(new InitializationInfo(), pointSourceFilePath, pointSampleInterval);
    //get record count to be generated
    long maxRecordCount = config.getRecordCount();
    long recordCount = 0;
    //prepare timer
    long startTS = System.currentTimeMillis();
    //prepare tweetIterator which acutally generates tweet 
    TweetMessageIterator tweetIterator = dataGenerator.new TweetMessageIterator(0, uidGenerator);
    FileOutputStream fos = null;
    try {
        //prepare output file
        fos = openOutputFile(config.getOutputFilePath());
        while (recordCount < maxRecordCount) {
            //get a tweet and append newline at the end
            String tweet = tweetIterator.next().toString() + "\n";
            //write to file
            fos.write(tweet.getBytes());
            recordCount++;
            if (recordCount % 1000000 == 0) {
                System.out.println("... generated " + recordCount + " records");
            }
        }
        System.out.println("Done: generated " + recordCount + " records in " + ((System.currentTimeMillis() - startTS) / 1000) + " in seconds!");
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        try {
            closeOutputFile(fos);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}