Examples with MutableLong org.apache.commons.lang.mutable.MutableLong used on opensource projects

Example 11 with MutableLong

use of org.apache.commons.lang.mutable.MutableLong in project apex-malhar by apache.

the class AbstractJMSInputOperator method messageConsumed.

/**
 * This method is called when a message is added to {@link #holdingBuffer} and can be overwritten by subclasses
 * if required. This is called by the JMS thread not Operator thread.
 *
 * @param message
 * @return message is accepted.
 * @throws javax.jms.JMSException
 */
protected boolean messageConsumed(Message message) throws JMSException {
    if (message.getJMSRedelivered() && pendingAck.contains(message.getJMSMessageID())) {
        counters.getCounter(CounterKeys.REDELIVERED).increment();
        LOG.warn("IGNORING: Redelivered Message {}", message.getJMSMessageID());
        return false;
    }
    pendingAck.add(message.getJMSMessageID());
    MutableLong receivedCt = counters.getCounter(CounterKeys.RECEIVED);
    receivedCt.increment();
    LOG.debug("message id: {} buffer size: {} received: {}", message.getJMSMessageID(), holdingBuffer.size(), receivedCt.longValue());
    return true;
}

Example 12 with MutableLong

use of org.apache.commons.lang.mutable.MutableLong in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    String lPhoneRange = conf.get(PHONE_RANGE_PROP, null);
    if (lPhoneRange != null) {
        String[] tokens = lPhoneRange.split("-");
        if (tokens.length != 2) {
            throw new IllegalArgumentException("Invalid range: " + lPhoneRange);
        }
        this.phoneRange = Range.between(Integer.parseInt(tokens[0]), Integer.parseInt(tokens[1]));
    }
    LOG.debug("Phone range {}", this.phoneRange);
    RandomEventGenerator phones = dag.addOperator("Receiver", RandomEventGenerator.class);
    phones.setMinvalue(this.phoneRange.getMinimum());
    phones.setMaxvalue(this.phoneRange.getMaximum());
    PhoneMovementGenerator movementGen = dag.addOperator("LocationFinder", PhoneMovementGenerator.class);
    dag.setAttribute(movementGen, OperatorContext.COUNTERS_AGGREGATOR, new BasicCounters.LongAggregator<MutableLong>());
    StatelessThroughputBasedPartitioner<PhoneMovementGenerator> partitioner = new StatelessThroughputBasedPartitioner<PhoneMovementGenerator>();
    partitioner.setCooldownMillis(conf.getLong(COOL_DOWN_MILLIS, 45000));
    partitioner.setMaximumEvents(conf.getLong(MAX_THROUGHPUT, 30000));
    partitioner.setMinimumEvents(conf.getLong(MIN_THROUGHPUT, 10000));
    dag.setAttribute(movementGen, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { partitioner }));
    dag.setAttribute(movementGen, OperatorContext.PARTITIONER, partitioner);
    // generate seed numbers
    Random random = new Random();
    int maxPhone = phoneRange.getMaximum() - phoneRange.getMinimum();
    int phonesToDisplay = conf.getInt(TOTAL_SEED_NOS, 10);
    for (int i = phonesToDisplay; i-- > 0; ) {
        int phoneNo = phoneRange.getMinimum() + random.nextInt(maxPhone + 1);
        LOG.info("seed no: " + phoneNo);
        movementGen.phoneRegister.add(phoneNo);
    }
    // done generating data
    LOG.info("Finished generating seed data.");
    URI uri = PubSubHelper.getURI(dag);
    PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator("LocationResults", new PubSubWebSocketOutputOperator<Object>());
    wsOut.setUri(uri);
    PubSubWebSocketInputOperator<Map<String, String>> wsIn = dag.addOperator("QueryLocation", new PubSubWebSocketInputOperator<Map<String, String>>());
    wsIn.setUri(uri);
    // default partitioning: first connected stream to movementGen will be partitioned
    dag.addStream("Phone-Data", phones.integer_data, movementGen.data);
    dag.addStream("Results", movementGen.locationQueryResult, wsOut.input);
    dag.addStream("Query", wsIn.outputPort, movementGen.phoneQuery);
}

Example 13 with MutableLong

use of org.apache.commons.lang.mutable.MutableLong in project apex-malhar by apache.

the class PhoneMovementGenerator method setup.

@Override
public void setup(OperatorContext context) {
    this.context = context;
    commandCounters.setCounter(CommandCounters.ADD, new MutableLong());
    commandCounters.setCounter(CommandCounters.ADD_RANGE, new MutableLong());
    commandCounters.setCounter(CommandCounters.DELETE, new MutableLong());
    commandCounters.setCounter(CommandCounters.CLEAR, new MutableLong());
}

Example 14 with MutableLong

use of org.apache.commons.lang.mutable.MutableLong in project knime-core by knime.

the class StatisticCalculator method evaluate.

/**
 * @param dataTable actual data table to compute the
 * @param exec execution context
 * @return a potential warnings message or <code>null</code>
 * @throws CanceledExecutionException if the user cancels the execution
 */
public String evaluate(final BufferedDataTable dataTable, final ExecutionContext exec) throws CanceledExecutionException {
    for (Statistic stat : m_statistics) {
        stat.beforeEvaluation(dataTable.size());
    }
    if (!m_colToSortOn.isEmpty()) {
        ColumnBufferedDataTableSorter columnDataTableSorter;
        try {
            columnDataTableSorter = new ColumnBufferedDataTableSorter(dataTable.getDataTableSpec(), dataTable.size(), m_colToSortOn.toArray(new String[m_colToSortOn.size()]));
        } catch (InvalidSettingsException e) {
            throw new RuntimeException("Error on initialize the sorting", e);
        }
        exec.setMessage("Sorting Data.");
        final Iterator<DataRow> it = dataTable.iterator();
        final MutableLong count = new MutableLong();
        final ExecutionContext evalProgress = exec.createSubExecutionContext(0.3);
        final int[] specMapping = createSpecMapping(dataTable.getSpec(), m_colToSortOn.toArray(new String[m_colToSortOn.size()]));
        columnDataTableSorter.sort(dataTable, exec.createSubExecutionContext(0.7), new SortingConsumer() {

            @Override
            public void consume(final DataRow defaultRow) {
                DataRow next = it.next();
                evalProgress.setProgress(count.longValue() / (double) dataTable.size(), "Processing Row: " + next.getKey());
                count.increment();
                for (Statistic stat : m_statistics) {
                    stat.consumeRow(new OverwritingRow(next, defaultRow, specMapping));
                }
            }
        });
    } else {
        exec.setMessage("Evaluating statistics.");
        long count = 0;
        for (DataRow currRow : dataTable) {
            exec.setProgress(count++ / (double) dataTable.size(), "Processing Row: " + currRow.getKey());
            for (Statistic stat : m_statistics) {
                stat.consumeRow(currRow);
            }
        }
    }
    StringBuilder warnings = new StringBuilder();
    for (Statistic stat : m_statistics) {
        String warningString = stat.finish();
        if (warningString != null) {
            warnings.append(warningString);
            warnings.append("\n");
        }
    }
    return warnings.length() > 0 ? warnings.toString() : null;
}

Example 15 with MutableLong

use of org.apache.commons.lang.mutable.MutableLong in project apex-core by apache.

the class RecordingsAgent method getRecordingInfoHelper.

private RecordingInfo getRecordingInfoHelper(String appId, String opId, String id, Set<String> containers) {
    RecordingInfo info = new RecordingInfo();
    info.id = id;
    info.appId = appId;
    info.operatorId = opId;
    BufferedReader br = null;
    IndexFileBufferedReader ifbr = null;
    try {
        String dir = getRecordingDirectory(appId, opId, id);
        if (dir == null) {
            throw new Exception("recording directory is null");
        }
        Path path = new Path(dir);
        JSONObject json;
        FileStatus fileStatus = stramAgent.getFileSystem().getFileStatus(path);
        HashMap<String, PortInfo> portMap = new HashMap<>();
        if (!fileStatus.isDirectory()) {
            throw new Exception(path + " is not a directory");
        }
        // META file processing
        br = new BufferedReader(new InputStreamReader(stramAgent.getFileSystem().open(new Path(dir, FSPartFileCollection.META_FILE))));
        String line;
        line = br.readLine();
        if (!line.equals("1.2")) {
            throw new Exception("Unexpected line: " + line);
        }
        line = br.readLine();
        json = new JSONObject(line);
        info.startTime = json.getLong("startTime");
        info.containerId = json.optString("containerId");
        info.properties = new HashMap<>();
        if (!StringUtils.isBlank(info.containerId) && !containers.contains(info.containerId)) {
            info.ended = true;
        }
        json = json.optJSONObject("properties");
        if (json != null) {
            @SuppressWarnings("unchecked") Iterator<String> keys = json.keys();
            while (keys.hasNext()) {
                String key = keys.next();
                // ugly 2 lines of code below since JSONObject.get(key).toString() doesn't give you json representation for plain strings
                String strValue = json.isNull(key) ? null : json.optString(key);
                info.properties.put(key, strValue != null ? strValue : new ObjectMapperString(json.get(key).toString()));
            }
        }
        info.ports = new ArrayList<>();
        while ((line = br.readLine()) != null) {
            PortInfo portInfo = new PortInfo();
            json = new JSONObject(line);
            portInfo.id = json.getInt("id");
            portInfo.name = json.getString("name");
            portInfo.type = json.getString("type");
            portInfo.streamName = json.getString("streamName");
            info.ports.add(portInfo);
            portMap.put(String.valueOf(portInfo.id), portInfo);
        }
        // INDEX file processing
        ifbr = new IndexFileBufferedReader(new InputStreamReader(stramAgent.getFileSystem().open(new Path(dir, FSPartFileCollection.INDEX_FILE))), dir);
        info.windowIdRanges = new ArrayList<>();
        long prevHiWindowId = -1;
        RecordingsIndexLine indexLine;
        while ((indexLine = (RecordingsIndexLine) ifbr.readIndexLine()) != null) {
            if (indexLine.isEndLine) {
                info.ended = true;
            } else {
                info.totalTuples += indexLine.tupleCount;
                for (Map.Entry<String, MutableLong> entry : indexLine.portTupleCount.entrySet()) {
                    PortInfo portInfo = portMap.get(entry.getKey());
                    if (portInfo == null) {
                        throw new Exception("port info does not exist for " + entry.getKey());
                    }
                    portInfo.tupleCount += entry.getValue().longValue();
                }
                for (TupleRecorder.Range r : indexLine.windowIdRanges) {
                    if (info.windowIdRanges.isEmpty()) {
                        TupleRecorder.Range range = new TupleRecorder.Range();
                        range.low = r.low;
                        info.windowIdRanges.add(range);
                    } else if (prevHiWindowId + 1 != r.low) {
                        TupleRecorder.Range range = info.windowIdRanges.get(info.windowIdRanges.size() - 1);
                        range.high = prevHiWindowId;
                        range = new TupleRecorder.Range();
                        range.low = r.low;
                        info.windowIdRanges.add(range);
                    }
                    prevHiWindowId = r.high;
                }
            }
        }
        if (!info.windowIdRanges.isEmpty()) {
            TupleRecorder.Range range = info.windowIdRanges.get(info.windowIdRanges.size() - 1);
            range.high = prevHiWindowId;
        }
    } catch (Exception ex) {
        LOG.warn("Cannot get recording info for app id {}: {}", appId, ex);
        return null;
    } finally {
        IOUtils.closeQuietly(ifbr);
        IOUtils.closeQuietly(br);
    }
    return info;
}