Examples with RandomEventGenerator org.apache.apex.malhar.lib.testbench.RandomEventGenerator used on opensource projects

Example 6 with RandomEventGenerator

use of org.apache.apex.malhar.lib.testbench.RandomEventGenerator in project apex-malhar by apache.

the class UniqueValueCountBenchmarkApplication method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration entries) {
    dag.setAttribute(dag.APPLICATION_NAME, "UniqueValueCountDemo");
    dag.setAttribute(dag.DEBUG, true);
    /* Generate random key-value pairs */
    RandomEventGenerator randGen = dag.addOperator("randomgen", new RandomEventGenerator());
    randGen.setMaxvalue(999999);
    randGen.setTuplesBlastIntervalMillis(50);
    dag.setAttribute(randGen, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<RandomEventGenerator>(3));
    /* Initialize with three partition to start with */
    UniqueCounter<Integer> uniqCount = dag.addOperator("uniqevalue", new UniqueCounter<Integer>());
    MapToKeyHashValuePairConverter<Integer, Integer> converter = dag.addOperator("converter", new MapToKeyHashValuePairConverter());
    dag.setAttribute(uniqCount, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<UniqueCounter<Integer>>(3));
    dag.setInputPortAttribute(uniqCount.data, Context.PortContext.PARTITION_PARALLEL, true);
    uniqCount.setCumulative(false);
    Counter counter = dag.addOperator("count", new Counter());
    ConsoleOutputOperator output = dag.addOperator("output", new ConsoleOutputOperator());
    dag.addStream("datain", randGen.integer_data, uniqCount.data);
    dag.addStream("convert", uniqCount.count, converter.input).setLocality(Locality.THREAD_LOCAL);
    dag.addStream("consoutput", converter.output, counter.input);
    dag.addStream("final", counter.output, output.input);
}

Example 7 with RandomEventGenerator

use of org.apache.apex.malhar.lib.testbench.RandomEventGenerator in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    String lPhoneRange = conf.get(PHONE_RANGE_PROP, null);
    if (lPhoneRange != null) {
        String[] tokens = lPhoneRange.split("-");
        if (tokens.length != 2) {
            throw new IllegalArgumentException("Invalid range: " + lPhoneRange);
        }
        this.phoneRange = Range.between(Integer.parseInt(tokens[0]), Integer.parseInt(tokens[1]));
    }
    LOG.debug("Phone range {}", this.phoneRange);
    RandomEventGenerator phones = dag.addOperator("Receiver", RandomEventGenerator.class);
    phones.setMinvalue(this.phoneRange.getMinimum());
    phones.setMaxvalue(this.phoneRange.getMaximum());
    PhoneMovementGenerator movementGen = dag.addOperator("LocationFinder", PhoneMovementGenerator.class);
    dag.setAttribute(movementGen, OperatorContext.COUNTERS_AGGREGATOR, new BasicCounters.LongAggregator<MutableLong>());
    StatelessThroughputBasedPartitioner<PhoneMovementGenerator> partitioner = new StatelessThroughputBasedPartitioner<PhoneMovementGenerator>();
    partitioner.setCooldownMillis(conf.getLong(COOL_DOWN_MILLIS, 45000));
    partitioner.setMaximumEvents(conf.getLong(MAX_THROUGHPUT, 30000));
    partitioner.setMinimumEvents(conf.getLong(MIN_THROUGHPUT, 10000));
    dag.setAttribute(movementGen, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { partitioner }));
    dag.setAttribute(movementGen, OperatorContext.PARTITIONER, partitioner);
    // generate seed numbers
    Random random = new Random();
    int maxPhone = phoneRange.getMaximum() - phoneRange.getMinimum();
    int phonesToDisplay = conf.getInt(TOTAL_SEED_NOS, 10);
    for (int i = phonesToDisplay; i-- > 0; ) {
        int phoneNo = phoneRange.getMinimum() + random.nextInt(maxPhone + 1);
        LOG.info("seed no: " + phoneNo);
        movementGen.phoneRegister.add(phoneNo);
    }
    // done generating data
    LOG.info("Finished generating seed data.");
    URI uri = PubSubHelper.getURI(dag);
    PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator("LocationResults", new PubSubWebSocketOutputOperator<Object>());
    wsOut.setUri(uri);
    PubSubWebSocketInputOperator<Map<String, String>> wsIn = dag.addOperator("QueryLocation", new PubSubWebSocketInputOperator<Map<String, String>>());
    wsIn.setUri(uri);
    // default partitioning: first connected stream to movementGen will be partitioned
    dag.addStream("Phone-Data", phones.integer_data, movementGen.data);
    dag.addStream("Results", movementGen.locationQueryResult, wsOut.input);
    dag.addStream("Query", wsIn.outputPort, movementGen.phoneQuery);
}

Example 8 with RandomEventGenerator

use of org.apache.apex.malhar.lib.testbench.RandomEventGenerator in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
    FibonacciOperator fib = dag.addOperator("FIB", FibonacciOperator.class);
    DefaultDelayOperator opDelay = dag.addOperator("opDelay", DefaultDelayOperator.class);
    StdoutOperator console = new StdoutOperator();
    console.setExtraOutputFileName(extraOutputFileName);
    dag.addOperator("console", console);
    dag.addStream("dummy_to_operator", rand.integer_data, fib.dummyInputPort);
    dag.addStream("operator_to_delay", fib.output, opDelay.input, console.input);
    dag.addStream("delay_to_operator", opDelay.output, fib.input);
}

Example 9 with RandomEventGenerator

use of org.apache.apex.malhar.lib.testbench.RandomEventGenerator in project apex-malhar by apache.

the class ApplicationAppData method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
    PiCalculateOperator calc = dag.addOperator("picalc", new PiCalculateOperator());
    dag.addStream("rand_calc", rand.integer_data, calc.input).setLocality(locality);
    AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());
    String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
    snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
    PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
    wsQuery.enableEmbeddedMode();
    snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);
    PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
    Operator.InputPort<String> queryResultPort = wsResult.input;
    NamedValueList<Object> adaptor = dag.addOperator("adaptor", new NamedValueList<Object>());
    ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("PiValues", calc.output, adaptor.inPort, console.input).setLocality(locality);
    ;
    dag.addStream("NamedPiValues", adaptor.outPort, snapshotServer.input);
    dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
}

Example 10 with RandomEventGenerator

use of org.apache.apex.malhar.lib.testbench.RandomEventGenerator in project apex-malhar by apache.

the class Calculator method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    /* keep generating random values between 0 and 30000 */
    RandomEventGenerator xyGenerator = dag.addOperator("GenerateX", RandomEventGenerator.class);
    /* calculate square of each of the values it receives */
    SquareCalculus squareOperator = dag.addOperator("SquareX", SquareCalculus.class);
    /* pair the consecutive values */
    AbstractAggregator<Integer> pairOperator = dag.addOperator("PairXY", new ArrayListAggregator<Integer>());
    Sigma<Integer> sumOperator = dag.addOperator("SumXY", new Sigma<Integer>());
    LogicalCompareToConstant<Integer> comparator = dag.addOperator("AnalyzeLocation", new LogicalCompareToConstant<Integer>());
    comparator.setConstant(30000 * 30000);
    Counter inCircle = dag.addOperator("CountInCircle", Counter.class);
    Counter inSquare = dag.addOperator("CountInSquare", Counter.class);
    Division division = dag.addOperator("Ratio", Division.class);
    MultiplyByConstant multiplication = dag.addOperator("InstantPI", MultiplyByConstant.class);
    multiplication.setMultiplier(4);
    RunningAverage average = dag.addOperator("AveragePI", new RunningAverage());
    ConsoleOutputOperator oper = dag.addOperator("Console", new ConsoleOutputOperator());
    dag.addStream("x", xyGenerator.integer_data, squareOperator.input);
    dag.addStream("sqr", squareOperator.integerResult, pairOperator.input);
    dag.addStream("x2andy2", pairOperator.output, sumOperator.input);
    dag.addStream("x2plusy2", sumOperator.integerResult, comparator.input, inSquare.input);
    dag.addStream("inCirclePoints", comparator.greaterThan, inCircle.input);
    dag.addStream("numerator", inCircle.output, division.numerator);
    dag.addStream("denominator", inSquare.output, division.denominator);
    dag.addStream("ratio", division.doubleQuotient, multiplication.input);
    dag.addStream("instantPi", multiplication.doubleProduct, average.input);
    dag.addStream("averagePi", average.doubleAverage, oper.input);
}