Examples with AtomicDouble com.google.common.util.concurrent.AtomicDouble used on opensource projects

Example 11 with AtomicDouble

use of com.google.common.util.concurrent.AtomicDouble in project Singularity by HubSpot.

the class SingularityUsageHelper method collectAgentUsage.

public void collectAgentUsage(SingularityAgent agent, long now, Map<String, RequestUtilization> utilizationPerRequestId, Map<String, RequestUtilization> previousUtilizations, Map<SingularityAgentUsage, List<TaskIdWithUsage>> overLoadedHosts, AtomicLong totalMemBytesUsed, AtomicLong totalMemBytesAvailable, AtomicDouble totalCpuUsed, AtomicDouble totalCpuAvailable, AtomicLong totalDiskBytesUsed, AtomicLong totalDiskBytesAvailable, boolean useShortTimeout) {
    Optional<Long> memoryMbTotal = Optional.empty();
    Optional<Double> cpusTotal = Optional.empty();
    Optional<Long> diskMbTotal = Optional.empty();
    long memoryMbReserved = 0;
    double cpuReserved = 0;
    long diskMbReserved = 0;
    long memoryBytesUsed = 0;
    double cpusUsed = 0;
    long diskMbUsed = 0;
    try {
        List<MesosTaskMonitorObject> allTaskUsage = mesosClient.getAgentResourceUsage(agent.getHost(), useShortTimeout);
        MesosAgentMetricsSnapshotObject agentMetricsSnapshot = mesosClient.getAgentMetricsSnapshot(agent.getHost());
        double systemMemTotalBytes = 0;
        double systemMemFreeBytes = 0;
        double systemLoad1Min = 0;
        double systemLoad5Min = 0;
        double systemLoad15Min = 0;
        double diskUsed = 0;
        double diskTotal = 0;
        double systemCpusTotal = 0;
        if (agentMetricsSnapshot != null) {
            systemMemTotalBytes = agentMetricsSnapshot.getSystemMemTotalBytes();
            systemMemFreeBytes = agentMetricsSnapshot.getSystemMemFreeBytes();
            systemLoad1Min = agentMetricsSnapshot.getSystemLoad1Min();
            systemLoad5Min = agentMetricsSnapshot.getSystemLoad5Min();
            systemLoad15Min = agentMetricsSnapshot.getSystemLoad15Min();
            diskUsed = agentMetricsSnapshot.getDiskUsed();
            diskTotal = agentMetricsSnapshot.getDiskTotal();
            systemCpusTotal = agentMetricsSnapshot.getSystemCpusTotal();
        }
        double systemLoad;
        switch(configuration.getMesosConfiguration().getScoreUsingSystemLoad()) {
            case LOAD_1:
                systemLoad = systemLoad1Min;
                break;
            case LOAD_15:
                systemLoad = systemLoad15Min;
                break;
            case LOAD_5:
            default:
                systemLoad = systemLoad5Min;
                break;
        }
        boolean overloadedForCpu = systemCpusTotal > 0 && systemLoad / systemCpusTotal > 1.0;
        boolean experiencingHighMemUsage = ((systemMemTotalBytes - systemMemFreeBytes) / systemMemTotalBytes) > configuration.getShuffleTasksWhenAgentMemoryUtilizationPercentageExceeds();
        List<TaskIdWithUsage> possibleTasksToShuffle = new ArrayList<>();
        Set<String> shuffleBlacklist = new HashSet<>(shuffleConfigurationManager.getShuffleBlocklist());
        for (MesosTaskMonitorObject taskUsage : allTaskUsage) {
            if (!taskUsage.getFrameworkId().equals(configuration.getMesosConfiguration().getFrameworkId())) {
                LOG.info("Skipping task {} from other framework {}", taskUsage.getSource(), taskUsage.getFrameworkId());
                continue;
            }
            String taskId = taskUsage.getSource();
            SingularityTaskId task;
            try {
                task = SingularityTaskId.valueOf(taskId);
            } catch (InvalidSingularityTaskIdException e) {
                LOG.warn("Couldn't get SingularityTaskId for {}", taskUsage);
                continue;
            }
            SingularityTaskUsage latestUsage = getUsage(taskUsage);
            List<SingularityTaskUsage> pastTaskUsages = usageManager.getTaskUsage(task);
            usageManager.saveSpecificTaskUsage(task, latestUsage);
            Optional<SingularityTask> maybeTask = taskManager.getTask(task);
            Optional<Resources> maybeResources = Optional.empty();
            if (maybeTask.isPresent()) {
                maybeResources = maybeTask.get().getTaskRequest().getPendingTask().getResources().isPresent() ? maybeTask.get().getTaskRequest().getPendingTask().getResources() : maybeTask.get().getTaskRequest().getDeploy().getResources();
                if (maybeResources.isPresent()) {
                    Resources taskResources = maybeResources.get();
                    double memoryMbReservedForTask = taskResources.getMemoryMb();
                    double cpuReservedForTask = taskResources.getCpus();
                    double diskMbReservedForTask = taskResources.getDiskMb();
                    memoryMbReserved += memoryMbReservedForTask;
                    cpuReserved += cpuReservedForTask;
                    diskMbReserved += diskMbReservedForTask;
                    updateRequestUtilization(utilizationPerRequestId, previousUtilizations.get(maybeTask.get().getTaskRequest().getRequest().getId()), pastTaskUsages, latestUsage, task, memoryMbReservedForTask, cpuReservedForTask, diskMbReservedForTask);
                }
            }
            memoryBytesUsed += latestUsage.getMemoryTotalBytes();
            diskMbUsed += latestUsage.getDiskTotalBytes();
            SingularityTaskCurrentUsage currentUsage = null;
            if (pastTaskUsages.isEmpty()) {
                Optional<SingularityTaskHistoryUpdate> maybeStartingUpdate = taskManager.getTaskHistoryUpdate(task, ExtendedTaskState.TASK_STARTING);
                if (maybeStartingUpdate.isPresent()) {
                    long startTimestamp = maybeStartingUpdate.get().getTimestamp();
                    double usedCpusSinceStart = latestUsage.getCpuSeconds() / TimeUnit.MILLISECONDS.toSeconds(latestUsage.getTimestamp() - startTimestamp);
                    currentUsage = new SingularityTaskCurrentUsage(latestUsage.getMemoryTotalBytes(), (long) taskUsage.getStatistics().getTimestamp() * 1000, usedCpusSinceStart, latestUsage.getDiskTotalBytes());
                    cpusUsed += usedCpusSinceStart;
                }
            } else {
                SingularityTaskUsage lastUsage = pastTaskUsages.get(pastTaskUsages.size() - 1);
                double taskCpusUsed = ((latestUsage.getCpuSeconds() - lastUsage.getCpuSeconds()) / TimeUnit.MILLISECONDS.toSeconds(latestUsage.getTimestamp() - lastUsage.getTimestamp()));
                currentUsage = new SingularityTaskCurrentUsage(latestUsage.getMemoryTotalBytes(), (long) taskUsage.getStatistics().getTimestamp() * 1000, taskCpusUsed, latestUsage.getDiskTotalBytes());
                cpusUsed += taskCpusUsed;
            }
            if (currentUsage != null && currentUsage.getCpusUsed() > 0) {
                if (isEligibleForShuffle(task, shuffleBlacklist)) {
                    Optional<SingularityTaskHistoryUpdate> maybeCleanupUpdate = taskManager.getTaskHistoryUpdate(task, ExtendedTaskState.TASK_CLEANING);
                    if (maybeCleanupUpdate.isPresent() && isTaskAlreadyCleanedUpForShuffle(maybeCleanupUpdate.get())) {
                        LOG.trace("Task {} already being cleaned up to spread cpu or mem usage, skipping", taskId);
                    } else {
                        if (maybeResources.isPresent()) {
                            possibleTasksToShuffle.add(new TaskIdWithUsage(task, maybeResources.get(), currentUsage));
                        }
                    }
                }
            }
        }
        if (!agent.getResources().isPresent() || !agent.getResources().get().getMemoryMegaBytes().isPresent() || !agent.getResources().get().getNumCpus().isPresent()) {
            LOG.debug("Could not find agent or resources for agent {}", agent.getId());
        } else {
            memoryMbTotal = Optional.of(agent.getResources().get().getMemoryMegaBytes().get().longValue());
            cpusTotal = Optional.of(agent.getResources().get().getNumCpus().get().doubleValue());
            diskMbTotal = Optional.of(agent.getResources().get().getDiskSpace().get());
        }
        SingularityAgentUsage agentUsage = new SingularityAgentUsage(cpusUsed, cpuReserved, cpusTotal, memoryBytesUsed, memoryMbReserved, memoryMbTotal, diskMbUsed, diskMbReserved, diskMbTotal, allTaskUsage.size(), now, systemMemTotalBytes, systemMemFreeBytes, systemCpusTotal, systemLoad1Min, systemLoad5Min, systemLoad15Min, diskUsed, diskTotal);
        if (overloadedForCpu || experiencingHighMemUsage) {
            overLoadedHosts.put(agentUsage, possibleTasksToShuffle);
        }
        if (agentUsage.getMemoryBytesTotal().isPresent() && agentUsage.getCpusTotal().isPresent()) {
            totalMemBytesUsed.getAndAdd((long) agentUsage.getMemoryBytesUsed());
            totalCpuUsed.getAndAdd(agentUsage.getCpusUsed());
            totalDiskBytesUsed.getAndAdd((long) agentUsage.getDiskBytesUsed());
            totalMemBytesAvailable.getAndAdd(agentUsage.getMemoryBytesTotal().get());
            totalCpuAvailable.getAndAdd(agentUsage.getCpusTotal().get());
            totalDiskBytesAvailable.getAndAdd(agentUsage.getDiskBytesTotal().get());
        }
        LOG.debug("Saving agent {} usage {}", agent.getHost(), agentUsage);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(agentUsage, agent.getId()));
    } catch (Throwable t) {
        String message = String.format("Could not get agent usage for host %s", agent.getHost());
        LOG.error(message, t);
        exceptionNotifier.notify(message, t);
    }
}

Example 12 with AtomicDouble

use of com.google.common.util.concurrent.AtomicDouble in project pinot by linkedin.

the class QuantileDigest method validate.

@VisibleForTesting
void validate() {
    final AtomicDouble sumOfWeights = new AtomicDouble();
    final AtomicInteger actualNodeCount = new AtomicInteger();
    final AtomicInteger actualNonZeroNodeCount = new AtomicInteger();
    if (root != null) {
        validateStructure(root);
        postOrderTraversal(root, new Callback() {

            @Override
            public boolean process(Node node) {
                sumOfWeights.addAndGet(node.weightedCount);
                actualNodeCount.incrementAndGet();
                if (node.weightedCount >= ZERO_WEIGHT_THRESHOLD) {
                    actualNonZeroNodeCount.incrementAndGet();
                }
                return true;
            }
        });
    }
    checkState(Math.abs(sumOfWeights.get() - weightedCount) < ZERO_WEIGHT_THRESHOLD, "Computed weight (%s) doesn't match summary (%s)", sumOfWeights.get(), weightedCount);
    checkState(actualNodeCount.get() == totalNodeCount, "Actual node count (%s) doesn't match summary (%s)", actualNodeCount.get(), totalNodeCount);
    checkState(actualNonZeroNodeCount.get() == nonZeroNodeCount, "Actual non-zero node count (%s) doesn't match summary (%s)", actualNonZeroNodeCount.get(), nonZeroNodeCount);
}

Example 13 with AtomicDouble

use of com.google.common.util.concurrent.AtomicDouble in project pinot by linkedin.

the class QuantileDigest method getHistogram.

/*
  * Get the exponentially-decayed approximate counts of values in multiple buckets. The elements in
  * the provided list denote the upper bound each of the buckets and must be sorted in ascending
  * order.
  *
  * The approximate count in each bucket is guaranteed to be within 2 * totalCount * maxError of
  * the real count.
  */
public List<Bucket> getHistogram(List<Long> bucketUpperBounds) {
    checkArgument(Ordering.natural().isOrdered(bucketUpperBounds), "buckets must be sorted in increasing order");
    final ImmutableList.Builder<Bucket> builder = ImmutableList.builder();
    final PeekingIterator<Long> iterator = Iterators.peekingIterator(bucketUpperBounds.iterator());
    final AtomicDouble sum = new AtomicDouble();
    final AtomicDouble lastSum = new AtomicDouble();
    // for computing weighed average of values in bucket
    final AtomicDouble bucketWeightedSum = new AtomicDouble();
    final double normalizationFactor = weight(TimeUnit.NANOSECONDS.toSeconds(ticker.read()));
    postOrderTraversal(root, new Callback() {

        @Override
        public boolean process(Node node) {
            while (iterator.hasNext() && iterator.peek() <= node.getUpperBound()) {
                double bucketCount = sum.get() - lastSum.get();
                Bucket bucket = new Bucket(bucketCount / normalizationFactor, bucketWeightedSum.get() / bucketCount);
                builder.add(bucket);
                lastSum.set(sum.get());
                bucketWeightedSum.set(0);
                iterator.next();
            }
            bucketWeightedSum.addAndGet(node.getMiddle() * node.weightedCount);
            sum.addAndGet(node.weightedCount);
            return iterator.hasNext();
        }
    });
    while (iterator.hasNext()) {
        double bucketCount = sum.get() - lastSum.get();
        Bucket bucket = new Bucket(bucketCount / normalizationFactor, bucketWeightedSum.get() / bucketCount);
        builder.add(bucket);
        iterator.next();
    }
    return builder.build();
}

Example 14 with AtomicDouble

use of com.google.common.util.concurrent.AtomicDouble in project deeplearning4j by deeplearning4j.

the class BarnesHutTsne method gradient.

@Override
public Gradient gradient() {
    if (yIncs == null)
        yIncs = zeros(Y.shape());
    if (gains == null)
        gains = ones(Y.shape());
    AtomicDouble sumQ = new AtomicDouble(0);
    /* Calculate gradient based on barnes hut approximation with positive and negative forces */
    INDArray posF = Nd4j.create(Y.shape());
    INDArray negF = Nd4j.create(Y.shape());
    if (tree == null)
        tree = new SpTree(Y);
    tree.computeEdgeForces(rows, cols, vals, N, posF);
    for (int n = 0; n < N; n++) tree.computeNonEdgeForces(n, theta, negF.slice(n), sumQ);
    INDArray dC = posF.subi(negF.divi(sumQ));
    Gradient ret = new DefaultGradient();
    ret.gradientForVariable().put(Y_GRAD, dC);
    return ret;
}

Example 15 with AtomicDouble

use of com.google.common.util.concurrent.AtomicDouble in project aic-praise by aic-sri-international.

the class AbstractUAI_to_HOGMv1_Translator method calculateCompressedEntries.

private static double calculateCompressedEntries(Expression compressedTableExpression) {
    AtomicDouble count = new AtomicDouble(0);
    visitCompressedTableEntries(compressedTableExpression, count);
    return count.doubleValue();
}