Examples with MachineKey org.apache.apex.examples.machinedata.data.MachineKey used on opensource projects

Example 6 with MachineKey

use of org.apache.apex.examples.machinedata.data.MachineKey in project apex-malhar by apache.

the class Application method addAverageCalculation.

/**
 * This function sets up the DAG for calculating the average
 *
 * @param dag  the DAG instance
 * @param conf the configuration instance
 * @return MachineInfoAveragingPrerequisitesOperator
 */
private MachineInfoAveragingPrerequisitesOperator addAverageCalculation(DAG dag, Configuration conf) {
    MachineInfoAveragingPrerequisitesOperator prereqAverageOper = dag.addOperator("Aggregator", MachineInfoAveragingPrerequisitesOperator.class);
    MachineInfoAveragingOperator averageOperator = dag.addOperator("AverageCalculator", MachineInfoAveragingOperator.class);
    RedisKeyValPairOutputOperator<MachineKey, Map<String, String>> redisAvgOperator = dag.addOperator("Persister", new RedisKeyValPairOutputOperator<MachineKey, Map<String, String>>());
    dag.addStream("Average", averageOperator.outputPort, redisAvgOperator.input);
    SmtpOutputOperator smtpOutputOperator = dag.addOperator("Alerter", new SmtpOutputOperator());
    dag.addStream("Aggregates", prereqAverageOper.outputPort, averageOperator.inputPort);
    dag.addStream("Alerts", averageOperator.smtpAlert, smtpOutputOperator.input);
    return prereqAverageOper;
}

Example 7 with MachineKey

use of org.apache.apex.examples.machinedata.data.MachineKey in project apex-malhar by apache.

the class InputReceiver method emitTuples.

@Override
public void emitTuples() {
    int count = 0;
    Calendar calendar = Calendar.getInstance();
    Date date = calendar.getTime();
    String timeKey = minuteDateFormat.format(date);
    String day = dayDateFormat.format(date);
    while (count < tupleBlastSize) {
        randomGen.setSeed(System.currentTimeMillis());
        int customerVal = genCustomerId();
        int productVal = genProductVer();
        int osVal = genOsVer();
        int software1Val = genSoftware1Ver();
        int software2Val = genSoftware2Ver();
        int software3Val = genSoftware3Ver();
        int deviceIdVal = genDeviceId();
        int cpuVal = genCpu(calendar);
        int ramVal = genRam(calendar);
        int hddVal = genHdd(calendar);
        MachineKey machineKey = new MachineKey(timeKey, day);
        machineKey.setCustomer(customerVal);
        machineKey.setProduct(productVal);
        machineKey.setOs(osVal);
        machineKey.setDeviceId(deviceIdVal);
        machineKey.setSoftware1(software1Val);
        machineKey.setSoftware2(software2Val);
        machineKey.setSoftware3(software3Val);
        MachineInfo machineInfo = new MachineInfo();
        machineInfo.setMachineKey(machineKey);
        machineInfo.setCpu(cpuVal);
        machineInfo.setRam(ramVal);
        machineInfo.setHdd(hddVal);
        outputInline.emit(machineInfo);
        count++;
    }
}

Example 8 with MachineKey

use of org.apache.apex.examples.machinedata.data.MachineKey in project apex-malhar by apache.

the class CalculatorOperator method endWindow.

@Override
public void endWindow() {
    if (computePercentile) {
        for (MachineKey machineKey : data.rowKeySet()) {
            Collections.sort(data.get(machineKey, ResourceType.CPU));
            Collections.sort(data.get(machineKey, ResourceType.RAM));
            Collections.sort(data.get(machineKey, ResourceType.HDD));
            Map<ResourceType, Double> percentileData = Maps.newHashMap();
            percentileData.put(ResourceType.CPU, getKthPercentile(data.get(machineKey, ResourceType.CPU)));
            percentileData.put(ResourceType.RAM, getKthPercentile(data.get(machineKey, ResourceType.RAM)));
            percentileData.put(ResourceType.HDD, getKthPercentile(data.get(machineKey, ResourceType.HDD)));
            percentileOutputPort.emit(new KeyValPair<>(machineKey, percentileData));
            for (ResourceType resourceType : percentileData.keySet()) {
                double percentileValue = percentileData.get(resourceType);
                if (percentileValue > percentileThreshold) {
                    emitAlert(resourceType, machineKey, percentileValue, "Percentile");
                }
            }
        }
    }
    if (computeSD) {
        for (MachineKey machineKey : data.rowKeySet()) {
            Map<ResourceType, Double> sdData = Maps.newHashMap();
            for (ResourceType resourceType : ResourceType.values()) {
                sdData.put(resourceType, getSD(data.get(machineKey, resourceType)));
            }
            sdOutputPort.emit(new KeyValPair<>(machineKey, sdData));
            for (ResourceType resourceType : sdData.keySet()) {
                double sdValue = sdData.get(resourceType);
                if (sdValue > sdThreshold) {
                    emitAlert(resourceType, machineKey, sdValue, "SD");
                }
            }
        }
    }
    if (computeMax) {
        for (MachineKey machineKey : data.rowKeySet()) {
            Map<ResourceType, Integer> maxData = Maps.newHashMap();
            maxData.put(ResourceType.CPU, Collections.max(data.get(machineKey, ResourceType.CPU)));
            maxData.put(ResourceType.RAM, Collections.max(data.get(machineKey, ResourceType.RAM)));
            maxData.put(ResourceType.HDD, Collections.max(data.get(machineKey, ResourceType.HDD)));
            maxOutputPort.emit(new KeyValPair<>(machineKey, maxData));
            for (ResourceType resourceType : maxData.keySet()) {
                double sdValue = maxData.get(resourceType).doubleValue();
                if (sdValue > maxThreshold) {
                    emitAlert(resourceType, machineKey, sdValue, "Max");
                }
            }
        }
    }
    data.clear();
}

Example 9 with MachineKey

use of org.apache.apex.examples.machinedata.data.MachineKey in project apex-malhar by apache.

the class CalculatorOperatorTest method testStandarDeviation.

public void testStandarDeviation(CalculatorOperator oper) {
    CollectorTestSink sortSink = new CollectorTestSink();
    oper.sdOutputPort.setSink(sortSink);
    Calendar calendar = Calendar.getInstance();
    Date date = calendar.getTime();
    String timeKey = minuteDateFormat.format(date);
    String day = calendar.get(Calendar.DAY_OF_MONTH) + "";
    Integer vs = new Integer(1);
    MachineKey mk = new MachineKey(timeKey, day, vs, vs, vs, vs, vs, vs, vs);
    oper.beginWindow(0);
    MachineInfo info = new MachineInfo(mk, 1, 1, 1);
    oper.dataPort.process(info);
    info.setCpu(2);
    oper.dataPort.process(info);
    info.setCpu(3);
    oper.dataPort.process(info);
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 1, sortSink.collectedTuples.size());
    for (Object o : sortSink.collectedTuples) {
        LOG.debug(o.toString());
        KeyValPair<TimeBucketKey, Map<ResourceType, Double>> keyValPair = (KeyValPair<TimeBucketKey, Map<ResourceType, Double>>) o;
        Assert.assertEquals("emitted value for 'cpu' was ", getSD(ImmutableList.of(1, 2, 3)), keyValPair.getValue().get(ResourceType.CPU), 0);
        Assert.assertEquals("emitted value for 'hdd' was ", getSD(ImmutableList.of(1, 1, 1)), keyValPair.getValue().get(ResourceType.HDD), 0);
        Assert.assertEquals("emitted value for 'ram' was ", getSD(ImmutableList.of(1, 1, 1)), keyValPair.getValue().get(ResourceType.RAM), 0);
    }
    LOG.debug("Done sd testing\n");
}

Example 10 with MachineKey

use of org.apache.apex.examples.machinedata.data.MachineKey in project apex-malhar by apache.

the class CalculatorOperatorTest method testPercentile.

public void testPercentile(CalculatorOperator oper) {
    CollectorTestSink sortSink = new CollectorTestSink();
    oper.percentileOutputPort.setSink(sortSink);
    oper.setKthPercentile(50);
    Calendar calendar = Calendar.getInstance();
    Date date = calendar.getTime();
    String timeKey = minuteDateFormat.format(date);
    String day = calendar.get(Calendar.DAY_OF_MONTH) + "";
    Integer vs = new Integer(1);
    MachineKey mk = new MachineKey(timeKey, day, vs, vs, vs, vs, vs, vs, vs);
    oper.beginWindow(0);
    MachineInfo info = new MachineInfo(mk, 1, 1, 1);
    oper.dataPort.process(info);
    info.setCpu(2);
    oper.dataPort.process(info);
    info.setCpu(3);
    oper.dataPort.process(info);
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 1, sortSink.collectedTuples.size());
    for (Object o : sortSink.collectedTuples) {
        LOG.debug(o.toString());
        KeyValPair<TimeBucketKey, Map<ResourceType, Double>> keyValPair = (KeyValPair<TimeBucketKey, Map<ResourceType, Double>>) o;
        Assert.assertEquals("emitted value for 'cpu' was ", 2.0, keyValPair.getValue().get(ResourceType.CPU), 0);
        Assert.assertEquals("emitted value for 'hdd' was ", 1.0, keyValPair.getValue().get(ResourceType.HDD), 0);
        Assert.assertEquals("emitted value for 'ram' was ", 1.0, keyValPair.getValue().get(ResourceType.RAM), 0);
    }
    LOG.debug("Done percentile testing\n");
}