Examples with StatelessThroughputBasedPartitioner org.apache.apex.malhar.lib.partitioner.StatelessThroughputBasedPartitioner used on opensource projects

Example 1 with StatelessThroughputBasedPartitioner

use of org.apache.apex.malhar.lib.partitioner.StatelessThroughputBasedPartitioner in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    String lPhoneRange = conf.get(PHONE_RANGE_PROP, null);
    if (lPhoneRange != null) {
        String[] tokens = lPhoneRange.split("-");
        if (tokens.length != 2) {
            throw new IllegalArgumentException("Invalid range: " + lPhoneRange);
        }
        this.phoneRange = Range.between(Integer.parseInt(tokens[0]), Integer.parseInt(tokens[1]));
    }
    LOG.debug("Phone range {}", this.phoneRange);
    RandomEventGenerator phones = dag.addOperator("Receiver", RandomEventGenerator.class);
    phones.setMinvalue(this.phoneRange.getMinimum());
    phones.setMaxvalue(this.phoneRange.getMaximum());
    PhoneMovementGenerator movementGen = dag.addOperator("LocationFinder", PhoneMovementGenerator.class);
    dag.setAttribute(movementGen, OperatorContext.COUNTERS_AGGREGATOR, new BasicCounters.LongAggregator<MutableLong>());
    StatelessThroughputBasedPartitioner<PhoneMovementGenerator> partitioner = new StatelessThroughputBasedPartitioner<PhoneMovementGenerator>();
    partitioner.setCooldownMillis(conf.getLong(COOL_DOWN_MILLIS, 45000));
    partitioner.setMaximumEvents(conf.getLong(MAX_THROUGHPUT, 30000));
    partitioner.setMinimumEvents(conf.getLong(MIN_THROUGHPUT, 10000));
    dag.setAttribute(movementGen, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { partitioner }));
    dag.setAttribute(movementGen, OperatorContext.PARTITIONER, partitioner);
    // generate seed numbers
    Random random = new Random();
    int maxPhone = phoneRange.getMaximum() - phoneRange.getMinimum();
    int phonesToDisplay = conf.getInt(TOTAL_SEED_NOS, 10);
    for (int i = phonesToDisplay; i-- > 0; ) {
        int phoneNo = phoneRange.getMinimum() + random.nextInt(maxPhone + 1);
        LOG.info("seed no: " + phoneNo);
        movementGen.phoneRegister.add(phoneNo);
    }
    // done generating data
    LOG.info("Finished generating seed data.");
    URI uri = PubSubHelper.getURI(dag);
    PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator("LocationResults", new PubSubWebSocketOutputOperator<Object>());
    wsOut.setUri(uri);
    PubSubWebSocketInputOperator<Map<String, String>> wsIn = dag.addOperator("QueryLocation", new PubSubWebSocketInputOperator<Map<String, String>>());
    wsIn.setUri(uri);
    // default partitioning: first connected stream to movementGen will be partitioned
    dag.addStream("Phone-Data", phones.integer_data, movementGen.data);
    dag.addStream("Results", movementGen.locationQueryResult, wsOut.input);
    dag.addStream("Query", wsIn.outputPort, movementGen.phoneQuery);
}