Examples with Counter org.apache.apex.malhar.lib.stream.Counter used on opensource projects

Example 1 with Counter

use of org.apache.apex.malhar.lib.stream.Counter in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    RandomKeyValGenerator randGen = dag.addOperator("RandomGenerator", new RandomKeyValGenerator());
    UniqueValueCount<Integer> valCount = dag.addOperator("UniqueCounter", new UniqueValueCount<Integer>());
    ConsoleOutputOperator consOut = dag.addOperator("Console", new ConsoleOutputOperator());
    StreamDuplicater<KeyValPair<Integer, Integer>> dup = dag.addOperator("Duplicator", new StreamDuplicater<KeyValPair<Integer, Integer>>());
    CountVerifier verifier = dag.addOperator("Verifier", new CountVerifier());
    ConsoleOutputOperator successOutput = dag.addOperator("Success", new ConsoleOutputOperator());
    successOutput.setStringFormat("Success %d");
    ConsoleOutputOperator failureOutput = dag.addOperator("Failure", new ConsoleOutputOperator());
    failureOutput.setStringFormat("Failure %d");
    Counter successcounter = dag.addOperator("SuccessCounter", new Counter());
    Counter failurecounter = dag.addOperator("FailureCounter", new Counter());
    dag.addStream("Events", randGen.outport, valCount.input);
    dag.addStream("Duplicates", valCount.output, dup.data);
    dag.addStream("Unverified", dup.out1, verifier.recIn);
    dag.addStream("EventCount", randGen.verport, verifier.trueIn);
    dag.addStream("Verified", verifier.successPort, successcounter.input);
    dag.addStream("Failed", verifier.failurePort, failurecounter.input);
    dag.addStream("SuccessCount", successcounter.output, successOutput.input);
    dag.addStream("FailedCount", failurecounter.output, failureOutput.input);
    dag.addStream("Output", dup.out2, consOut.input);
}

Example 2 with Counter

use of org.apache.apex.malhar.lib.stream.Counter in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    // set app name
    dag.setAttribute(DAG.APPLICATION_NAME, "Logstream Application");
    dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, 1000);
    ConsoleOutputOperator mysqlConsole = dag.addOperator("MysqlConsole", ConsoleOutputOperator.class);
    ConsoleOutputOperator syslogConsole = dag.addOperator("SyslogConsole", ConsoleOutputOperator.class);
    ConsoleOutputOperator systemConsole = dag.addOperator("SystemlogConsole", ConsoleOutputOperator.class);
    LogScoreOperator logScoreOperator = dag.addOperator("logscore", new LogScoreOperator());
    /*
     * Read log file messages from a messaging system (Redis, RabbitMQ, etc)
     * Typically one message equates to a single line in a log file, but in
     * some cases may be multiple lines such as java stack trace, etc.
     */
    // Get logs from RabbitMQ
    RabbitMQLogsInputOperator apacheLogInput = dag.addOperator("ApacheLogInput", RabbitMQLogsInputOperator.class);
    RabbitMQLogsInputOperator mysqlLogInput = dag.addOperator("MysqlLogInput", RabbitMQLogsInputOperator.class);
    RabbitMQLogsInputOperator syslogLogInput = dag.addOperator("SyslogLogInput", RabbitMQLogsInputOperator.class);
    RabbitMQLogsInputOperator systemLogInput = dag.addOperator("SystemLogInput", RabbitMQLogsInputOperator.class);
    /*
     * Convert incoming JSON structures to flattened map objects
     */
    JsonByteArrayOperator apacheLogJsonToMap = dag.addOperator("ApacheLogJsonToMap", JsonByteArrayOperator.class);
    dag.addStream("apache_convert_type", apacheLogInput.outputPort, apacheLogJsonToMap.input);
    JsonByteArrayOperator mysqlLogJsonToMap = dag.addOperator("MysqlLogJsonToMap", JsonByteArrayOperator.class);
    dag.addStream("mysql_convert_type", mysqlLogInput.outputPort, mysqlLogJsonToMap.input);
    JsonByteArrayOperator syslogLogJsonToMap = dag.addOperator("SyslogLogJsonToMap", JsonByteArrayOperator.class);
    dag.addStream("syslog_convert_type", syslogLogInput.outputPort, syslogLogJsonToMap.input);
    JsonByteArrayOperator systemLogJsonToMap = dag.addOperator("SystemLogJsonToMap", JsonByteArrayOperator.class);
    dag.addStream("system_convert_type", systemLogInput.outputPort, systemLogJsonToMap.input);
    // operator for tuple counter
    Counter apacheLogCounter = dag.addOperator("ApacheLogCounter", new Counter());
    /*
     * operators for sum of bytes
     */
    SumItemFromMapOperator<String, Object> apacheLogBytesSum = dag.addOperator("ApacheLogBytesSum", new SumItemFromMapOperator<String, Object>());
    apacheLogBytesSum.setSumDimension("bytes");
    /*
     * opeartor filter 404 logs
     */
    SelectOperator filter404 = getFilteredMessagesOperator("Filter404", dag);
    /*
     * Explode dimensions based on log types ( apache, mysql, syslog, etc)
     */
    DimensionTimeBucketSumOperator apacheDimensionOperator = getApacheDimensionTimeBucketSumOperator("ApacheLogDimension", dag);
    DimensionTimeBucketSumOperator apacheFilteredDimensionOperator = getFilteredApacheDimensionTimeBucketSumOperator("ApacheFilteredLogDimension", dag);
    dag.addStream("apache_dimension_in", apacheLogJsonToMap.outputFlatMap, apacheDimensionOperator.in, apacheLogCounter.input, apacheLogBytesSum.mapInput, filter404.inport);
    dag.addStream("apache_filtered_dimension_in", filter404.outport, apacheFilteredDimensionOperator.in);
    DimensionTimeBucketSumOperator mysqlDimensionOperator = getMysqlDimensionTimeBucketSumOperator("MysqlLogDimension", dag);
    dag.addStream("mysql_dimension_in", mysqlLogJsonToMap.outputFlatMap, mysqlDimensionOperator.in);
    DimensionTimeBucketSumOperator syslogDimensionOperator = getSyslogDimensionTimeBucketSumOperator("syslogLogDimension", dag);
    dag.addStream("syslog_dimension_in", syslogLogJsonToMap.outputFlatMap, syslogDimensionOperator.in);
    // DimensionTimeBucketSumOperator systemDimensionOperator = getSystemDimensionTimeBucketSumOperator("systemLogDimension", dag);
    // dag.addStream("system_dimension_in", systemLogJsonToMap.outputMap, systemDimensionOperator.in);
    /*
     * Calculate average, min, max, etc from dimensions ( based on log types )
     */
    // aggregating over sliding window
    MultiWindowDimensionAggregation apacheMultiWindowAggCountOpr = getApacheAggregationCountOper("apache_sliding_window_count", dag);
    MultiWindowDimensionAggregation apacheMultiWindowAggSumOpr = getApacheAggregationSumOper("apache_sliding_window_sum", dag);
    dag.addStream("apache_dimension_out", apacheDimensionOperator.out, apacheMultiWindowAggCountOpr.data, apacheMultiWindowAggSumOpr.data);
    MultiWindowDimensionAggregation apacheFilteredMultiWindowAggCountOpr = getFilteredApacheAggregationCountOper("apache_filtered_sliding_window_count", dag);
    dag.addStream("apache_filtered_dimension_out", apacheFilteredDimensionOperator.out, apacheFilteredMultiWindowAggCountOpr.data);
    MultiWindowDimensionAggregation mysqlMultiWindowAggOpr = getMysqlAggregationOper("mysql_sliding_window", dag);
    dag.addStream("mysql_dimension_out", mysqlDimensionOperator.out, mysqlMultiWindowAggOpr.data);
    MultiWindowDimensionAggregation syslogMultiWindowAggOpr = getSyslogAggregationOper("syslog_sliding_window", dag);
    dag.addStream("syslog_dimension_out", syslogDimensionOperator.out, syslogMultiWindowAggOpr.data);
    // MultiWindowDimensionAggregation systemMultiWindowAggOpr = getSystemAggregationOper("system_sliding_window", dag);
    // dag.addStream("system_dimension_out", systemDimensionOperator.out, systemMultiWindowAggOpr.data);
    // adding top N operator
    TopN<String, DimensionObject<String>> apacheTopNCountOpr = dag.addOperator("apache_topN_count", new TopN<String, DimensionObject<String>>());
    apacheTopNCountOpr.setN(10);
    TopN<String, DimensionObject<String>> apacheFilteredTopNCountOpr = dag.addOperator("apache_filtered_topN_count", new TopN<String, DimensionObject<String>>());
    apacheFilteredTopNCountOpr.setN(10);
    TopN<String, DimensionObject<String>> apacheTopNSumOpr = dag.addOperator("apache_topN_sum", new TopN<String, DimensionObject<String>>());
    apacheTopNSumOpr.setN(10);
    TopN<String, DimensionObject<String>> mysqlTopNOpr = dag.addOperator("mysql_topN", new TopN<String, DimensionObject<String>>());
    mysqlTopNOpr.setN(5);
    TopN<String, DimensionObject<String>> syslogTopNOpr = dag.addOperator("syslog_topN", new TopN<String, DimensionObject<String>>());
    syslogTopNOpr.setN(5);
    // TopN<String, DimensionObject<String>> systemTopNOpr = dag.addOperator("system_topN", new TopN<String, DimensionObject<String>>());
    // systemTopNOpr.setN(5);
    /*
     * Analytics Engine
     */
    dag.addStream("ApacheLogScoreCount", apacheMultiWindowAggCountOpr.output, apacheTopNCountOpr.data, logScoreOperator.apacheLogs);
    dag.addStream("ApacheFilteredLogScoreCount", apacheFilteredMultiWindowAggCountOpr.output, apacheFilteredTopNCountOpr.data);
    dag.addStream("ApacheLogScoreSum", apacheMultiWindowAggSumOpr.output, apacheTopNSumOpr.data);
    dag.addStream("MysqlLogScore", mysqlMultiWindowAggOpr.output, mysqlTopNOpr.data, logScoreOperator.mysqlLogs);
    dag.addStream("SyslogLogScore", syslogMultiWindowAggOpr.output, syslogTopNOpr.data, logScoreOperator.syslogLogs);
    // dag.addStream("SystemLogScore", systemLogJsonToMap.outputMap, logScoreOperator.systemLogs);
    /*
     * Alerts
     */
    // TODO
    /*
     * Console output for debugging purposes
     */
    // ConsoleOutputOperator console = dag.addOperator("console", ConsoleOutputOperator.class);
    // dag.addStream("topn_output", apacheTopNOpr.top);entry.getValue()
    /*
     * write to redis for siteops
     */
    // prepare operators to convert output streams to redis output format
    AggregationsToRedisOperator<Integer, Integer> apacheLogCounterToRedis = dag.addOperator("ApacheLogCounterToRedis", new AggregationsToRedisOperator<Integer, Integer>());
    apacheLogCounterToRedis.setKeyIndex(11);
    AggregationsToRedisOperator<String, String> apacheLogBytesSumToRedis = dag.addOperator("ApacheLogBytesSumToRedis", new AggregationsToRedisOperator<String, String>());
    apacheLogBytesSumToRedis.setKeyIndex(9);
    AggregationsToRedisOperator<String, DimensionObject<String>> topNCountToRedis = getApacheTopNToRedisOperatorCountAggregation("topNCountToRedis", dag);
    AggregationsToRedisOperator<String, DimensionObject<String>> filteredTopNCountToRedis = getFilteredApacheTopNToRedisOperatorCountAggregation("filteredTopNCountToRedis", dag);
    AggregationsToRedisOperator<String, DimensionObject<String>> topNSumToRedis = getApacheTopNToRedisOperatorSumAggregation("topNSumToRedis", dag);
    // convert ouputs for redis output operator and send output to web socket output
    dag.addStream("apache_log_counter", apacheLogCounter.output, apacheLogCounterToRedis.valueInput);
    dag.addStream("apache_log_bytes_sum", apacheLogBytesSum.output, apacheLogBytesSumToRedis.valueInput);
    dag.addStream("topn_redis_count", apacheTopNCountOpr.top, topNCountToRedis.multiWindowDimensionInput, wsOutput(dag, "apacheTopAggrs"));
    dag.addStream("topn_redis_sum", apacheTopNSumOpr.top, topNSumToRedis.multiWindowDimensionInput, wsOutput(dag, "apacheTopSumAggrs"));
    dag.addStream("filtered_topn_redis_count", apacheFilteredTopNCountOpr.top, filteredTopNCountToRedis.multiWindowDimensionInput);
    // redis output operators
    RedisKeyValPairOutputOperator<String, String> redisOutTotalCount = dag.addOperator("RedisOutTotalCount", new RedisKeyValPairOutputOperator<String, String>());
    redisOutTotalCount.getStore().setDbIndex(15);
    RedisKeyValPairOutputOperator<String, String> redisOutTotalSumBytes = dag.addOperator("RedisOutTotalSumBytes", new RedisKeyValPairOutputOperator<String, String>());
    redisOutTotalSumBytes.getStore().setDbIndex(15);
    RedisMapOutputOperator<String, String> redisOutTopNCount = dag.addOperator("RedisOutTopNCount", new RedisMapOutputOperator<String, String>());
    redisOutTopNCount.getStore().setDbIndex(15);
    RedisMapOutputOperator<String, String> redisOutTopNSum = dag.addOperator("RedisOutTopNSum", new RedisMapOutputOperator<String, String>());
    redisOutTopNSum.getStore().setDbIndex(15);
    RedisMapOutputOperator<String, String> redisOutFilteredTopNSum = dag.addOperator("RedisOutFilteredTopNSum", new RedisMapOutputOperator<String, String>());
    redisOutFilteredTopNSum.getStore().setDbIndex(15);
    // redis output streams
    dag.addStream("apache_log_counter_to_redis", apacheLogCounterToRedis.keyValPairOutput, redisOutTotalCount.input);
    dag.addStream("apache_log_bytes_sum_to_redis", apacheLogBytesSumToRedis.keyValPairOutput, redisOutTotalSumBytes.input);
    dag.addStream("apache_log_dimension_counter_to_redis", topNCountToRedis.keyValueMapOutput, redisOutTopNCount.input);
    dag.addStream("apache_log_dimension_sum_to_redis", topNSumToRedis.keyValueMapOutput, redisOutTopNSum.input);
    dag.addStream("apache_log_filtered_dimension_count_to_redis", filteredTopNCountToRedis.keyValueMapOutput, redisOutFilteredTopNSum.input);
    /*
     * Websocket output to UI from calculated aggregations
     */
    dag.addStream("MysqlTopAggregations", mysqlTopNOpr.top, wsOutput(dag, "mysqlTopAggrs"), mysqlConsole.input);
    dag.addStream("SyslogTopAggregations", syslogTopNOpr.top, wsOutput(dag, "syslogTopAggrs"), syslogConsole.input);
    dag.addStream("SystemData", systemLogJsonToMap.outputMap, wsOutput(dag, "systemData"), logScoreOperator.systemLogs, systemConsole.input);
}

Example 3 with Counter

use of org.apache.apex.malhar.lib.stream.Counter in project apex-malhar by apache.

the class UniqueValueCountBenchmarkApplication method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration entries) {
    dag.setAttribute(dag.APPLICATION_NAME, "UniqueValueCountDemo");
    dag.setAttribute(dag.DEBUG, true);
    /* Generate random key-value pairs */
    RandomEventGenerator randGen = dag.addOperator("randomgen", new RandomEventGenerator());
    randGen.setMaxvalue(999999);
    randGen.setTuplesBlastIntervalMillis(50);
    dag.setAttribute(randGen, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<RandomEventGenerator>(3));
    /* Initialize with three partition to start with */
    UniqueCounter<Integer> uniqCount = dag.addOperator("uniqevalue", new UniqueCounter<Integer>());
    MapToKeyHashValuePairConverter<Integer, Integer> converter = dag.addOperator("converter", new MapToKeyHashValuePairConverter());
    dag.setAttribute(uniqCount, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<UniqueCounter<Integer>>(3));
    dag.setInputPortAttribute(uniqCount.data, Context.PortContext.PARTITION_PARALLEL, true);
    uniqCount.setCumulative(false);
    Counter counter = dag.addOperator("count", new Counter());
    ConsoleOutputOperator output = dag.addOperator("output", new ConsoleOutputOperator());
    dag.addStream("datain", randGen.integer_data, uniqCount.data);
    dag.addStream("convert", uniqCount.count, converter.input).setLocality(Locality.THREAD_LOCAL);
    dag.addStream("consoutput", converter.output, counter.input);
    dag.addStream("final", counter.output, output.input);
}

Example 4 with Counter

use of org.apache.apex.malhar.lib.stream.Counter in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration entries) {
    /* Generate random key-value pairs */
    RandomKeysGenerator randGen = dag.addOperator("randomgen", new RandomKeysGenerator());
    /* Initialize with three partition to start with */
    // UniqueCount1 uniqCount = dag.addOperator("uniqevalue", new UniqueCount1());
    UniqueCounter<Integer> uniqCount = dag.addOperator("uniqevalue", new UniqueCounter<Integer>());
    MapToKeyHashValuePairConverter<Integer, Integer> converter = dag.addOperator("converter", new MapToKeyHashValuePairConverter());
    uniqCount.setCumulative(false);
    dag.setAttribute(uniqCount, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<UniqueCounter<Integer>>(3));
    CountVerifier<Integer> verifier = dag.addOperator("verifier", new CountVerifier<Integer>());
    StreamDuplicater<KeyHashValPair<Integer, Integer>> dup = dag.addOperator("dup", new StreamDuplicater<KeyHashValPair<Integer, Integer>>());
    ConsoleOutputOperator output = dag.addOperator("output", new ConsoleOutputOperator());
    ConsoleOutputOperator successOutput = dag.addOperator("successoutput", new ConsoleOutputOperator());
    successOutput.setStringFormat("Success %d");
    ConsoleOutputOperator failureOutput = dag.addOperator("failureoutput", new ConsoleOutputOperator());
    failureOutput.setStringFormat("Failure %d");
    // success and failure counters.
    Counter successcounter = dag.addOperator("successcounter", new Counter());
    Counter failurecounter = dag.addOperator("failurecounter", new Counter());
    dag.addStream("datain", randGen.outPort, uniqCount.data);
    dag.addStream("dataverification0", randGen.verificationPort, verifier.in1);
    dag.addStream("convert", uniqCount.count, converter.input).setLocality(Locality.THREAD_LOCAL);
    dag.addStream("split", converter.output, dup.data);
    dag.addStream("consoutput", dup.out1, output.input);
    dag.addStream("dataverification1", dup.out2, verifier.in2);
    dag.addStream("successc", verifier.successPort, successcounter.input);
    dag.addStream("failurec", verifier.failurePort, failurecounter.input);
    dag.addStream("succconsoutput", successcounter.output, successOutput.input);
    dag.addStream("failconsoutput", failurecounter.output, failureOutput.input);
}

Example 5 with Counter

use of org.apache.apex.malhar.lib.stream.Counter in project apex-malhar by apache.

the class Calculator method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    /* keep generating random values between 0 and 30000 */
    RandomEventGenerator xyGenerator = dag.addOperator("GenerateX", RandomEventGenerator.class);
    /* calculate square of each of the values it receives */
    SquareCalculus squareOperator = dag.addOperator("SquareX", SquareCalculus.class);
    /* pair the consecutive values */
    AbstractAggregator<Integer> pairOperator = dag.addOperator("PairXY", new ArrayListAggregator<Integer>());
    Sigma<Integer> sumOperator = dag.addOperator("SumXY", new Sigma<Integer>());
    LogicalCompareToConstant<Integer> comparator = dag.addOperator("AnalyzeLocation", new LogicalCompareToConstant<Integer>());
    comparator.setConstant(30000 * 30000);
    Counter inCircle = dag.addOperator("CountInCircle", Counter.class);
    Counter inSquare = dag.addOperator("CountInSquare", Counter.class);
    Division division = dag.addOperator("Ratio", Division.class);
    MultiplyByConstant multiplication = dag.addOperator("InstantPI", MultiplyByConstant.class);
    multiplication.setMultiplier(4);
    RunningAverage average = dag.addOperator("AveragePI", new RunningAverage());
    ConsoleOutputOperator oper = dag.addOperator("Console", new ConsoleOutputOperator());
    dag.addStream("x", xyGenerator.integer_data, squareOperator.input);
    dag.addStream("sqr", squareOperator.integerResult, pairOperator.input);
    dag.addStream("x2andy2", pairOperator.output, sumOperator.input);
    dag.addStream("x2plusy2", sumOperator.integerResult, comparator.input, inSquare.input);
    dag.addStream("inCirclePoints", comparator.greaterThan, inCircle.input);
    dag.addStream("numerator", inCircle.output, division.numerator);
    dag.addStream("denominator", inSquare.output, division.denominator);
    dag.addStream("ratio", division.doubleQuotient, multiplication.input);
    dag.addStream("instantPi", multiplication.doubleProduct, average.input);
    dag.addStream("averagePi", average.doubleAverage, oper.input);
}