Examples with RandomMapOutput org.apache.apex.benchmark.RandomMapOutput used on opensource projects

Example 1 with RandomMapOutput

use of org.apache.apex.benchmark.RandomMapOutput in project apex-malhar by apache.

the class RubyOperatorBenchmarkApplication method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
    rand.setMaxvalue(3000);
    rand.setTuplesBlast(120);
    RandomMapOutput randMap = dag.addOperator("randMap", new RandomMapOutput());
    randMap.setKey("val");
    RubyOperator ruby = dag.addOperator("ruby", new RubyOperator());
    String setupScript = "def square(val)\n";
    setupScript += "  return val*val\nend\n";
    ruby.addSetupScript(setupScript);
    ruby.setInvoke("square");
    ruby.setPassThru(true);
    ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
    dag.getMeta(console).getMeta(console.input).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
    dag.getMeta(ruby).getMeta(ruby.result).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
    dag.addStream("rand_randMap", rand.integer_data, randMap.input).setLocality(Locality.THREAD_LOCAL);
    dag.addStream("randMap_ruby", randMap.map_data, ruby.inBindings).setLocality(locality);
    dag.addStream("ruby_console", ruby.result, console.input).setLocality(locality);
}

Example 2 with RandomMapOutput

use of org.apache.apex.benchmark.RandomMapOutput in project apex-malhar by apache.

the class HiveMapInsertBenchmarkingApp method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    HiveStore store = new HiveStore();
    store.setDatabaseUrl(conf.get("dt.application.HiveMapInsertBenchmarkingApp.operator.HiveOperator.store.dbUrl"));
    store.setConnectionProperties(conf.get("dt.application.HiveMapInsertBenchmarkingApp.operator.HiveOperator.store.connectionProperties"));
    store.setFilepath(conf.get("dt.application.HiveMapInsertBenchmarkingApp.operator.HiveOperator.store.filepath"));
    try {
        hiveInitializeMapDatabase(store, conf.get("dt.application.HiveMapInsertBenchmarkingApp.operator.HiveOperator.tablename"), ":");
    } catch (SQLException ex) {
        LOG.debug(ex.getMessage());
    }
    dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, 1000);
    RandomEventGenerator eventGenerator = dag.addOperator("EventGenerator", RandomEventGenerator.class);
    RandomMapOutput mapGenerator = dag.addOperator("MapGenerator", RandomMapOutput.class);
    dag.setAttribute(eventGenerator, PortContext.QUEUE_CAPACITY, 10000);
    dag.setAttribute(mapGenerator, PortContext.QUEUE_CAPACITY, 10000);
    HiveOperator hiveInsert = dag.addOperator("HiveOperator", new HiveOperator());
    hiveInsert.setStore(store);
    FSRollingMapTestImpl rollingMapFsWriter = dag.addOperator("RollingFsMapWriter", new FSRollingMapTestImpl());
    rollingMapFsWriter.setFilePath(store.filepath);
    ArrayList<String> hivePartitionColumns = new ArrayList<String>();
    hivePartitionColumns.add("dt");
    hiveInsert.setHivePartitionColumns(hivePartitionColumns);
    dag.addStream("EventGenerator2Map", eventGenerator.integer_data, mapGenerator.input);
    dag.addStream("MapGenerator2HdfsOutput", mapGenerator.map_data, rollingMapFsWriter.input);
    dag.addStream("FsWriter2Hive", rollingMapFsWriter.outputPort, hiveInsert.input);
}