Examples with DescriptiveStatistics org.apache.commons.math3.stat.descriptive.DescriptiveStatistics used on opensource projects

Example 61 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project plc4x by apache.

the class ManualS7PlcDriverMT method parallelScheduledLoop.

@Test
public void parallelScheduledLoop() throws InterruptedException {
    int period = 5;
    PlcDriverManager plcDriverManager = new PooledPlcDriverManager();
    ScheduledExecutorService executorService = Executors.newScheduledThreadPool(10);
    DescriptiveStatistics statistics1 = new DescriptiveStatistics();
    DescriptiveStatistics statistics2 = new DescriptiveStatistics();
    int numberOfRuns = 1000;
    AtomicInteger counter1 = new AtomicInteger(0);
    AtomicInteger counter2 = new AtomicInteger(0);
    executorService.scheduleAtFixedRate(() -> {
        // System.out.println("Run: " + counter.get());
        double timeNs = runSingleRequest(plcDriverManager);
        statistics1.addValue(timeNs);
        if (counter1.getAndIncrement() >= numberOfRuns) {
            executorService.shutdown();
        }
    }, 0, period, TimeUnit.MILLISECONDS);
    executorService.scheduleAtFixedRate(() -> {
        // System.out.println("Run: " + counter.get());
        double timeNs = runSingleRequest(plcDriverManager);
        statistics2.addValue(timeNs);
        if (counter2.getAndIncrement() >= numberOfRuns) {
            executorService.shutdown();
        }
    }, 0, period, TimeUnit.MILLISECONDS);
    executorService.awaitTermination(100, TimeUnit.SECONDS);
    System.out.println("Statistics 1");
    printStatistics(statistics1);
    System.out.println("Statistics 2");
    printStatistics(statistics2);
}

Example 62 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project plc4x by apache.

the class TriggeredScraperImpl method start.

/**
 * Start the scraping.
 */
// ToDo code-refactoring and improved testing --> PLC4X-90
@Override
public void start() {
    // Schedule all jobs
    LOGGER.info("Starting jobs...");
    // start iterating over all available jobs
    for (ScrapeJob job : jobs) {
        // iterate over all source the jobs shall performed on
        for (Map.Entry<String, String> sourceEntry : job.getSourceConnections().entrySet()) {
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("Register task for job {} for conn {} ({}) at rate {} ms", job.getJobName(), sourceEntry.getKey(), sourceEntry.getValue(), job.getScrapeRate());
            }
            // create the regarding triggered scraper task
            TriggeredScraperTask triggeredScraperTask;
            try {
                triggeredScraperTask = new TriggeredScraperTask(driverManager, job.getJobName(), sourceEntry.getKey(), sourceEntry.getValue(), job.getFields(), futureTimeOut, executorService, resultHandler, (TriggeredScrapeJobImpl) job, triggerCollector);
                // Add task to internal list
                if (LOGGER.isInfoEnabled()) {
                    LOGGER.info("Task {} added to scheduling", triggeredScraperTask);
                }
                registerTaskMBean(triggeredScraperTask);
                tasks.put(job, triggeredScraperTask);
                ScheduledFuture<?> future = scheduler.scheduleAtFixedRate(triggeredScraperTask, 0, job.getScrapeRate(), TimeUnit.MILLISECONDS);
                // Store the handle for stopping, etc.
                scraperTaskMap.put(triggeredScraperTask, future);
            } catch (ScraperException e) {
                LOGGER.warn("Error executing the job {} for conn {} ({}) at rate {} ms", job.getJobName(), sourceEntry.getKey(), sourceEntry.getValue(), job.getScrapeRate(), e);
            }
        }
    }
    // Add statistics tracker
    statisticsLogger = scheduler.scheduleAtFixedRate(() -> {
        for (Map.Entry<ScrapeJob, ScraperTask> entry : tasks.entries()) {
            DescriptiveStatistics statistics = entry.getValue().getLatencyStatistics();
            String msg = String.format(Locale.ENGLISH, "Job statistics (%s, %s) number of requests: %d (%d success, %.1f %% failed, %.1f %% too slow), min latency: %.2f ms, mean latency: %.2f ms, median: %.2f ms", entry.getValue().getJobName(), entry.getValue().getConnectionAlias(), entry.getValue().getRequestCounter(), entry.getValue().getSuccessfullRequestCounter(), entry.getValue().getPercentageFailed(), statistics.apply(new PercentageAboveThreshold(entry.getKey().getScrapeRate() * 1e6)), statistics.getMin() * 1e-6, statistics.getMean() * 1e-6, statistics.getPercentile(50) * 1e-6);
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug(msg);
            }
        }
    }, 1_000, 1_000, TimeUnit.MILLISECONDS);
}

Example 63 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project plc4x by apache.

the class ScraperImpl method start.

@Override
public void start() {
    // Schedule all jobs
    LOGGER.info("Starting jobs...");
    jobs.stream().flatMap(job -> job.getSourceConnections().entrySet().stream().map(entry -> Triple.of(job, entry.getKey(), entry.getValue()))).forEach(tuple -> {
        LOGGER.debug("Register task for job {} for conn {} ({}) at rate {} ms", tuple.getLeft().getJobName(), tuple.getMiddle(), tuple.getRight(), tuple.getLeft().getScrapeRate());
        ScraperTask task = new ScraperTaskImpl(driverManager, tuple.getLeft().getJobName(), tuple.getMiddle(), tuple.getRight(), tuple.getLeft().getFields(), 1_000, handlerPool, resultHandler);
        // Add task to internal list
        tasks.put(tuple.getLeft(), task);
        ScheduledFuture<?> future = scheduler.scheduleAtFixedRate(task, 0, tuple.getLeft().getScrapeRate(), TimeUnit.MILLISECONDS);
        // Store the handle for stopping, etc.
        futures.put(task, future);
    });
    // Add statistics tracker
    scheduler.scheduleAtFixedRate(() -> {
        for (Map.Entry<ScrapeJob, ScraperTask> entry : tasks.entries()) {
            DescriptiveStatistics statistics = entry.getValue().getLatencyStatistics();
            String msg = String.format(Locale.ENGLISH, "Job statistics (%s, %s) number of requests: %d (%d success, %.1f %% failed, %.1f %% too slow), min latency: %.2f ms, mean latency: %.2f ms, median: %.2f ms", entry.getValue().getJobName(), entry.getValue().getConnectionAlias(), entry.getValue().getRequestCounter(), entry.getValue().getSuccessfullRequestCounter(), entry.getValue().getPercentageFailed(), statistics.apply(new PercentageAboveThreshold(entry.getKey().getScrapeRate() * 1e6)), statistics.getMin() * 1e-6, statistics.getMean() * 1e-6, statistics.getPercentile(50) * 1e-6);
            LOGGER.debug(msg);
        }
    }, 1_000, 1_000, TimeUnit.MILLISECONDS);
}

Example 64 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project datawave by NationalSecurityAgency.

the class RangeStreamScanner method scannerInvariant.

protected int scannerInvariant(final Iterator<Entry<Key, Value>> iter) {
    PeekingIterator<Entry<Key, Value>> kvIter = new PeekingIterator<>(iter);
    int retrievalCount = 0;
    Entry<Key, Value> myEntry;
    String currentDay = null;
    if (null != prevDay) {
        try {
            if (log.isTraceEnabled())
                log.trace("Attempting to insert " + prevDay);
            if (!resultQueue.offer(prevDay, 1, TimeUnit.SECONDS)) {
                return 0;
            }
            prevDay = null;
        } catch (InterruptedException e) {
            return 0;
        }
    }
    // produces stats for us, so we don't have to!
    DescriptiveStatistics stats = new DescriptiveStatistics();
    writeLock.lock();
    try {
        while (kvIter.hasNext()) {
            Entry<Key, Value> currentKeyValue = kvIter.peek();
            // become a pass-through if we've seen an unexpected key.
            if (seenUnexpectedKey) {
                currentQueue.add(trimTrailingUnderscore(currentKeyValue));
                break;
            }
            if (null == currentDay) {
                if (log.isTraceEnabled()) {
                    log.trace("it's a new day!");
                    log.trace("adding " + currentKeyValue.getKey() + " to queue because it matches" + currentDay);
                }
                currentDay = getDay(currentKeyValue.getKey());
                currentQueue.add(trimTrailingUnderscore(currentKeyValue));
                lastSeenKey = kvIter.next().getKey();
            } else {
                String nextKeysDay = getDay(currentKeyValue.getKey());
                if (currentDay.equals(nextKeysDay)) {
                    if (log.isTraceEnabled()) {
                        log.trace("adding " + currentKeyValue.getKey() + " to queue because it matches" + currentDay);
                    }
                    IndexInfo info = readInfoFromValue(currentKeyValue.getValue());
                    if (log.isTraceEnabled()) {
                        log.trace("adding count of " + info.count());
                    }
                    stats.addValue(info.count());
                    if (currentQueue.size() <= shardsPerDayThreshold || stats.getPercentile(50) < MAX_MEDIAN) {
                        if (log.isTraceEnabled()) {
                            log.trace("adding our stats are " + stats.getPercentile(50) + " on " + currentQueue.size());
                        }
                        currentQueue.add(trimTrailingUnderscore(currentKeyValue));
                    } else {
                        if (log.isTraceEnabled()) {
                            log.trace("breaking because our stats are " + stats.getPercentile(50) + " on " + currentQueue.size());
                        }
                        break;
                    }
                    lastSeenKey = kvIter.next().getKey();
                } else {
                    int dequeueCount = dequeue();
                    retrievalCount += dequeueCount;
                    int queueSize = currentQueue.size();
                    dequeue(true);
                    currentDay = null;
                    if (dequeueCount != queueSize || retrievalCount <= Math.ceil(maxResults * 1.5)) {
                        break;
                    }
                }
            }
        }
        if (currentQueue.size() >= shardsPerDayThreshold && stats.getPercentile(50) > MAX_MEDIAN) {
            Entry<Key, Value> top = currentQueue.poll();
            Key topKey = top.getKey();
            if (log.isTraceEnabled())
                log.trace(topKey + " for " + currentDay + " exceeds limit of " + shardsPerDayThreshold + " with " + currentQueue.size());
            Key newKey = new Key(topKey.getRow(), topKey.getColumnFamily(), new Text(currentDay), topKey.getColumnVisibility(), topKey.getTimestamp());
            Value newValue = writeInfoToValue();
            myEntry = Maps.immutableEntry(newKey, newValue);
            lastSeenKey = newKey;
            try {
                if (!resultQueue.offer(myEntry, 1, TimeUnit.SECONDS)) {
                    if (log.isTraceEnabled()) {
                        log.trace("could not add day! converting " + myEntry + " to " + prevDay);
                    }
                    prevDay = myEntry;
                }
            } catch (InterruptedException exception) {
                prevDay = myEntry;
            }
            currentQueue.clear();
        } else {
            retrievalCount += dequeue();
        }
    } finally {
        writeLock.unlock();
    }
    return retrievalCount;
}

Example 65 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project azkaban by azkaban.

the class InMemoryMetricEmitter method statBasedSelectMetricHistory.

/**
 * filter snapshots using statistically significant points only
 *
 * @param selectedLists list of snapshots
 */
private void statBasedSelectMetricHistory(final LinkedList<InMemoryHistoryNode> selectedLists) throws ClassCastException {
    logger.debug("selecting snapshots which are far away from mean value");
    final DescriptiveStatistics descStats = getDescriptiveStatistics(selectedLists);
    final Double mean = descStats.getMean();
    final Double std = descStats.getStandardDeviation();
    final Iterator<InMemoryHistoryNode> ite = selectedLists.iterator();
    while (ite.hasNext()) {
        final InMemoryHistoryNode currentNode = ite.next();
        final double value = ((Number) currentNode.getValue()).doubleValue();
        // remove all elements which lies in 95% value band
        if (value < mean + this.standardDeviationFactor * std && value > mean - this.standardDeviationFactor * std) {
            ite.remove();
        }
    }
}