Search in sources :

Example 1 with TweetMessageIterator

use of org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessageIterator in project asterixdb by apache.

the class SyntheticDataGeneratorForSpatialIndexEvaluation method main.

public static void main(String[] args) throws Exception {
    SyntheticDataGeneratorConfig config = new SyntheticDataGeneratorConfig();
    CmdLineParser clp = new CmdLineParser(config);
    try {
        clp.parseArgument(args);
    } catch (CmdLineException e) {
        System.err.println(e.getMessage());
        clp.printUsage(System.err);
        System.exit(1);
    }
    //prepare data generator
    GULongIDGenerator uidGenerator = new GULongIDGenerator(config.getPartitionId(), (byte) (0));
    String pointSourceFilePath = config.getPointSourceFile();
    int pointSampleInterval = config.getpointSamplingInterval();
    DataGeneratorForSpatialIndexEvaluation dataGenerator = new DataGeneratorForSpatialIndexEvaluation(new InitializationInfo(), pointSourceFilePath, pointSampleInterval);
    //get record count to be generated
    long maxRecordCount = config.getRecordCount();
    long recordCount = 0;
    //prepare timer
    long startTS = System.currentTimeMillis();
    //prepare tweetIterator which acutally generates tweet 
    TweetMessageIterator tweetIterator = dataGenerator.new TweetMessageIterator(0, uidGenerator);
    FileOutputStream fos = null;
    try {
        //prepare output file
        fos = openOutputFile(config.getOutputFilePath());
        while (recordCount < maxRecordCount) {
            //get a tweet and append newline at the end
            String tweet = tweetIterator.next().toString() + "\n";
            //write to file
            fos.write(tweet.getBytes());
            recordCount++;
            if (recordCount % 1000000 == 0) {
                System.out.println("... generated " + recordCount + " records");
            }
        }
        System.out.println("Done: generated " + recordCount + " records in " + ((System.currentTimeMillis() - startTS) / 1000) + " in seconds!");
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        try {
            closeOutputFile(fos);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
Also used : CmdLineParser(org.kohsuke.args4j.CmdLineParser) TweetMessageIterator(org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessageIterator) GULongIDGenerator(org.apache.asterix.tools.external.data.GULongIDGenerator) IOException(java.io.IOException) CmdLineException(org.kohsuke.args4j.CmdLineException) IOException(java.io.IOException) FileOutputStream(java.io.FileOutputStream) DataGeneratorForSpatialIndexEvaluation(org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation) InitializationInfo(org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.InitializationInfo) CmdLineException(org.kohsuke.args4j.CmdLineException)

Aggregations

FileOutputStream (java.io.FileOutputStream)1 IOException (java.io.IOException)1 DataGeneratorForSpatialIndexEvaluation (org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation)1 InitializationInfo (org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.InitializationInfo)1 TweetMessageIterator (org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessageIterator)1 GULongIDGenerator (org.apache.asterix.tools.external.data.GULongIDGenerator)1 CmdLineException (org.kohsuke.args4j.CmdLineException)1 CmdLineParser (org.kohsuke.args4j.CmdLineParser)1