Examples with MutableLong org.apache.commons.lang.mutable.MutableLong used on opensource projects

Example 36 with MutableLong

use of org.apache.commons.lang.mutable.MutableLong in project apex-core by apache.

the class RecordingsAgent method parseIndexLine.

@Override
protected RecordingsIndexLine parseIndexLine(String line) throws JSONException {
    RecordingsIndexLine info = new RecordingsIndexLine();
    if (line.startsWith("E")) {
        info.isEndLine = true;
        return info;
    }
    line = line.trim();
    info.windowIdRanges = new ArrayList<>();
    info.portTupleCount = new HashMap<>();
    int cursor = 2;
    int cursor2 = line.indexOf(':', cursor);
    info.partFile = line.substring(cursor, cursor2);
    cursor = cursor2 + 1;
    cursor2 = line.indexOf(':', cursor);
    String timeRange = line.substring(cursor, cursor2);
    String[] tmp = timeRange.split("-");
    info.fromTime = Long.valueOf(tmp[0]);
    info.toTime = Long.valueOf(tmp[1]);
    cursor = cursor2 + 1;
    cursor2 = line.indexOf(':', cursor);
    if (cursor2 < 0) {
        info.tupleCount = Long.valueOf(line.substring(cursor));
        return info;
    }
    info.tupleCount = Long.valueOf(line.substring(cursor, cursor2));
    cursor = cursor2 + 1;
    if (!line.substring(cursor, cursor + 2).equals("T:")) {
        return info;
    }
    cursor += 2;
    cursor2 = line.indexOf(':', cursor);
    String windowRangesString = line.substring(cursor, cursor2);
    String[] windowRanges = windowRangesString.split(",");
    for (String windowRange : windowRanges) {
        String[] hilow = windowRange.split("-");
        long low = Long.valueOf(hilow[0]);
        long hi = Long.valueOf(hilow[1]);
        info.windowIdRanges.add(new TupleRecorder.Range(low, hi));
    }
    cursor = cursor2 + 1;
    cursor2 = line.indexOf(':', cursor);
    int size = Integer.valueOf(line.substring(cursor, cursor2));
    cursor = cursor2 + 1;
    cursor2 = cursor + size;
    JSONObject json = new JSONObject(line.substring(cursor, cursor2));
    Iterator<?> keys = json.keys();
    while (keys.hasNext()) {
        String portIndex = (String) keys.next();
        long tupleCount = json.getLong(portIndex);
        if (!info.portTupleCount.containsKey(portIndex)) {
            info.portTupleCount.put(portIndex, new MutableLong(tupleCount));
        } else {
            info.portTupleCount.get(portIndex).add(tupleCount);
        }
    }
    return info;
}

Example 37 with MutableLong

use of org.apache.commons.lang.mutable.MutableLong in project druid by druid-io.

the class StreamAppenderator method persistAll.

@Override
public ListenableFuture<Object> persistAll(@Nullable final Committer committer) {
    throwPersistErrorIfExists();
    final Map<String, Integer> currentHydrants = new HashMap<>();
    final List<Pair<FireHydrant, SegmentIdWithShardSpec>> indexesToPersist = new ArrayList<>();
    int numPersistedRows = 0;
    long bytesPersisted = 0L;
    MutableLong totalHydrantsCount = new MutableLong();
    MutableLong totalHydrantsPersisted = new MutableLong();
    final long totalSinks = sinks.size();
    for (Map.Entry<SegmentIdWithShardSpec, Sink> entry : sinks.entrySet()) {
        final SegmentIdWithShardSpec identifier = entry.getKey();
        final Sink sink = entry.getValue();
        if (sink == null) {
            throw new ISE("No sink for identifier: %s", identifier);
        }
        final List<FireHydrant> hydrants = Lists.newArrayList(sink);
        totalHydrantsCount.add(hydrants.size());
        currentHydrants.put(identifier.toString(), hydrants.size());
        numPersistedRows += sink.getNumRowsInMemory();
        bytesPersisted += sink.getBytesInMemory();
        final int limit = sink.isWritable() ? hydrants.size() - 1 : hydrants.size();
        // gather hydrants that have not been persisted:
        for (FireHydrant hydrant : hydrants.subList(0, limit)) {
            if (!hydrant.hasSwapped()) {
                log.debug("Hydrant[%s] hasn't persisted yet, persisting. Segment[%s]", hydrant, identifier);
                indexesToPersist.add(Pair.of(hydrant, identifier));
                totalHydrantsPersisted.add(1);
            }
        }
        if (sink.swappable()) {
            // It is swappable. Get the old one to persist it and create a new one:
            indexesToPersist.add(Pair.of(sink.swap(), identifier));
            totalHydrantsPersisted.add(1);
        }
    }
    log.debug("Submitting persist runnable for dataSource[%s]", schema.getDataSource());
    final Object commitMetadata = committer == null ? null : committer.getMetadata();
    final Stopwatch runExecStopwatch = Stopwatch.createStarted();
    final Stopwatch persistStopwatch = Stopwatch.createStarted();
    AtomicLong totalPersistedRows = new AtomicLong(numPersistedRows);
    final ListenableFuture<Object> future = persistExecutor.submit(new Callable<Object>() {

        @Override
        public Object call() throws IOException {
            try {
                for (Pair<FireHydrant, SegmentIdWithShardSpec> pair : indexesToPersist) {
                    metrics.incrementRowOutputCount(persistHydrant(pair.lhs, pair.rhs));
                }
                if (committer != null) {
                    log.debug("Committing metadata[%s] for sinks[%s].", commitMetadata, Joiner.on(", ").join(currentHydrants.entrySet().stream().map(entry -> StringUtils.format("%s:%d", entry.getKey(), entry.getValue())).collect(Collectors.toList())));
                    committer.run();
                    try {
                        commitLock.lock();
                        final Map<String, Integer> commitHydrants = new HashMap<>();
                        final Committed oldCommit = readCommit();
                        if (oldCommit != null) {
                            // merge current hydrants with existing hydrants
                            commitHydrants.putAll(oldCommit.getHydrants());
                        }
                        commitHydrants.putAll(currentHydrants);
                        writeCommit(new Committed(commitHydrants, commitMetadata));
                    } finally {
                        commitLock.unlock();
                    }
                }
                log.info("Flushed in-memory data with commit metadata [%s] for segments: %s", commitMetadata, indexesToPersist.stream().map(itp -> itp.rhs.asSegmentId().toString()).distinct().collect(Collectors.joining(", ")));
                log.info("Persisted stats: processed rows: [%d], persisted rows[%d], sinks: [%d], total fireHydrants (across sinks): [%d], persisted fireHydrants (across sinks): [%d]", rowIngestionMeters.getProcessed(), totalPersistedRows.get(), totalSinks, totalHydrantsCount.longValue(), totalHydrantsPersisted.longValue());
                // return null if committer is null
                return commitMetadata;
            } catch (IOException e) {
                metrics.incrementFailedPersists();
                throw e;
            } finally {
                metrics.incrementNumPersists();
                metrics.incrementPersistTimeMillis(persistStopwatch.elapsed(TimeUnit.MILLISECONDS));
                persistStopwatch.stop();
            }
        }
    });
    final long startDelay = runExecStopwatch.elapsed(TimeUnit.MILLISECONDS);
    metrics.incrementPersistBackPressureMillis(startDelay);
    if (startDelay > WARN_DELAY) {
        log.warn("Ingestion was throttled for [%,d] millis because persists were pending.", startDelay);
    }
    runExecStopwatch.stop();
    resetNextFlush();
    // NB: The rows are still in memory until they're done persisting, but we only count rows in active indexes.
    rowsCurrentlyInMemory.addAndGet(-numPersistedRows);
    bytesCurrentlyInMemory.addAndGet(-bytesPersisted);
    log.info("Persisted rows[%,d] and (estimated) bytes[%,d]", numPersistedRows, bytesPersisted);
    return future;
}