Examples with Sum org.apache.commons.math3.stat.descriptive.summary.Sum used on opensource projects

Example 26 with Sum

use of org.apache.commons.math3.stat.descriptive.summary.Sum in project cruise-control by linkedin.

the class ClusterModel method sanityCheck.

/**
 * (1) Check whether each load in the cluster contains exactly the number of windows defined by the Load.
 * (2) Check whether sum of loads in the cluster / rack / broker / replica are consistent with each other.
 */
public void sanityCheck() {
    // SANITY CHECK #1: Each load in the cluster must contain exactly the number of windows defined by the Load.
    Map<String, Integer> errorMsgAndNumWindows = new HashMap<>();
    int expectedNumWindows = _load.numWindows();
    // Check leadership loads.
    for (Map.Entry<Integer, Load> entry : _potentialLeadershipLoadByBrokerId.entrySet()) {
        int brokerId = entry.getKey();
        Load load = entry.getValue();
        if (load.numWindows() != expectedNumWindows && broker(brokerId).replicas().size() != 0) {
            errorMsgAndNumWindows.put("Leadership(" + brokerId + ")", load.numWindows());
        }
    }
    // Check rack loads.
    for (Rack rack : _racksById.values()) {
        if (rack.load().numWindows() != expectedNumWindows && rack.replicas().size() != 0) {
            errorMsgAndNumWindows.put("Rack(id:" + rack.id() + ")", rack.load().numWindows());
        }
        // Check the host load.
        for (Host host : rack.hosts()) {
            if (host.load().numWindows() != expectedNumWindows && host.replicas().size() != 0) {
                errorMsgAndNumWindows.put("Host(id:" + host.name() + ")", host.load().numWindows());
            }
            // Check broker loads.
            for (Broker broker : rack.brokers()) {
                if (broker.load().numWindows() != expectedNumWindows && broker.replicas().size() != 0) {
                    errorMsgAndNumWindows.put("Broker(id:" + broker.id() + ")", broker.load().numWindows());
                }
                // Check replica loads.
                for (Replica replica : broker.replicas()) {
                    if (replica.load().numWindows() != expectedNumWindows) {
                        errorMsgAndNumWindows.put("Replica(id:" + replica.topicPartition() + "-" + broker.id() + ")", replica.load().numWindows());
                    }
                }
            }
        }
    }
    StringBuilder exceptionMsg = new StringBuilder();
    for (Map.Entry<String, Integer> entry : errorMsgAndNumWindows.entrySet()) {
        exceptionMsg.append(String.format("[%s: %d]%n", entry.getKey(), entry.getValue()));
    }
    if (exceptionMsg.length() > 0) {
        throw new IllegalArgumentException("Loads must have all have " + expectedNumWindows + " windows. Following " + "loads violate this constraint with specified number of windows: " + exceptionMsg);
    }
    // SANITY CHECK #2: Sum of loads in the cluster / rack / broker / replica must be consistent with each other.
    String prologueErrorMsg = "Inconsistent load distribution.";
    // Check equality of sum of the replica load to their broker load for each resource.
    for (Broker broker : brokers()) {
        for (Resource resource : Resource.values()) {
            double sumOfReplicaUtilization = 0.0;
            for (Replica replica : broker.replicas()) {
                sumOfReplicaUtilization += replica.load().expectedUtilizationFor(resource);
            }
            if (AnalyzerUtils.compare(sumOfReplicaUtilization, broker.load().expectedUtilizationFor(resource), resource) != 0) {
                throw new IllegalArgumentException(prologueErrorMsg + " Broker utilization for " + resource + " is different " + "from the total replica utilization in the broker with id: " + broker.id() + ". Sum of the replica utilization: " + sumOfReplicaUtilization + ", broker utilization: " + broker.load().expectedUtilizationFor(resource));
            }
        }
    }
    // Check equality of sum of the broker load to their rack load for each resource.
    Map<Resource, Double> sumOfRackUtilizationByResource = new HashMap<>();
    for (Rack rack : _racksById.values()) {
        Map<Resource, Double> sumOfHostUtilizationByResource = new HashMap<>();
        for (Host host : rack.hosts()) {
            for (Resource resource : Resource.values()) {
                sumOfHostUtilizationByResource.putIfAbsent(resource, 0.0);
                double sumOfBrokerUtilization = 0.0;
                for (Broker broker : host.brokers()) {
                    sumOfBrokerUtilization += broker.load().expectedUtilizationFor(resource);
                }
                Double hostUtilization = host.load().expectedUtilizationFor(resource);
                if (AnalyzerUtils.compare(sumOfBrokerUtilization, hostUtilization, resource) != 0) {
                    throw new IllegalArgumentException(prologueErrorMsg + " Host utilization for " + resource + " is different " + "from the total broker utilization in the host : " + host.name() + ". Sum of the brokers: " + sumOfBrokerUtilization + ", host utilization: " + hostUtilization);
                }
                sumOfHostUtilizationByResource.put(resource, sumOfHostUtilizationByResource.get(resource) + hostUtilization);
            }
        }
        // Check equality of sum of the host load to the rack load for each resource.
        for (Map.Entry<Resource, Double> entry : sumOfHostUtilizationByResource.entrySet()) {
            Resource resource = entry.getKey();
            double sumOfHostsUtil = entry.getValue();
            sumOfRackUtilizationByResource.putIfAbsent(resource, 0.0);
            Double rackUtilization = rack.load().expectedUtilizationFor(resource);
            if (AnalyzerUtils.compare(rackUtilization, sumOfHostsUtil, resource) != 0) {
                throw new IllegalArgumentException(prologueErrorMsg + " Rack utilization for " + resource + " is different " + "from the total host utilization in rack" + rack.id() + " . Sum of the hosts: " + sumOfHostsUtil + ", rack utilization: " + rack.load().expectedUtilizationFor(resource));
            }
            sumOfRackUtilizationByResource.put(resource, sumOfRackUtilizationByResource.get(resource) + sumOfHostUtilizationByResource.get(resource));
        }
    }
    // Check equality of sum of the rack load to the cluster load for each resource.
    for (Map.Entry<Resource, Double> entry : sumOfRackUtilizationByResource.entrySet()) {
        Resource resource = entry.getKey();
        double sumOfRackUtil = entry.getValue();
        if (AnalyzerUtils.compare(_load.expectedUtilizationFor(resource), sumOfRackUtil, resource) != 0) {
            throw new IllegalArgumentException(prologueErrorMsg + " Cluster utilization for " + resource + " is different " + "from the total rack utilization in the cluster. Sum of the racks: " + sumOfRackUtil + ", cluster utilization: " + _load.expectedUtilizationFor(resource));
        }
    }
    // Check equality of the sum of the leadership load to the sum of the load of leader at each broker.
    for (Broker broker : brokers()) {
        double sumOfLeaderOfReplicaUtilization = 0.0;
        for (Replica replica : broker.replicas()) {
            sumOfLeaderOfReplicaUtilization += partition(replica.topicPartition()).leader().load().expectedUtilizationFor(Resource.NW_OUT);
        }
        if (AnalyzerUtils.compare(sumOfLeaderOfReplicaUtilization, _potentialLeadershipLoadByBrokerId.get(broker.id()).expectedUtilizationFor(Resource.NW_OUT), Resource.NW_OUT) != 0) {
            throw new IllegalArgumentException(prologueErrorMsg + " Leadership utilization for " + Resource.NW_OUT + " is different from the total utilization leader of replicas in the broker" + " with id: " + broker.id() + " Expected: " + sumOfLeaderOfReplicaUtilization + " Received: " + _potentialLeadershipLoadByBrokerId.get(broker.id()).expectedUtilizationFor(Resource.NW_OUT) + ".");
        }
        for (Resource resource : Resource.values()) {
            if (resource == Resource.CPU) {
                continue;
            }
            double leaderSum = broker.leaderReplicas().stream().mapToDouble(r -> r.load().expectedUtilizationFor(resource)).sum();
            double cachedLoad = broker.leadershipLoadForNwResources().expectedUtilizationFor(resource);
            if (AnalyzerUtils.compare(leaderSum, cachedLoad, resource) != 0) {
                throw new IllegalArgumentException(prologueErrorMsg + " Leadership load for resource " + resource + " is " + cachedLoad + " but recomputed sum is " + leaderSum + ".");
            }
        }
    }
}

Example 27 with Sum

use of org.apache.commons.math3.stat.descriptive.summary.Sum in project imagingbook-common by imagingbook.

the class Matrix method main.

// --------------------------------------------------------------------------
public static void main(String[] args) {
    float[][] A = { { -1, 2, 3 }, { 4, 5, 6 }, { 7, 8, 9 } };
    System.out.println("A = \n" + toString(A));
    System.out.println("A rows = " + getNumberOfRows(A));
    System.out.println("A columns = " + getNumberOfColumns(A));
    int row = 1;
    int column = 2;
    System.out.println("A[1][2] = " + A[row][column]);
    System.out.println("det=" + determinant3x3(A));
    float[][] Ai = inverse(A);
    toString(Ai);
    double[][] B = { { -1, 2, 3 }, { 4, 5, 6 } };
    System.out.println("B = \n" + toString(B));
    System.out.println("B rows = " + getNumberOfRows(B));
    System.out.println("B columns = " + getNumberOfColumns(B));
    double[] rowSum = sumRows(B);
    System.out.println("B sum of rows = " + toString(rowSum));
    double[] colSum = sumColumns(B);
    System.out.println("B sum of cols = " + toString(colSum));
    PrintPrecision.set(5);
    double[][] C = new double[2][3];
    System.out.println("C rows = " + getNumberOfRows(C));
    System.out.println("C columns = " + getNumberOfColumns(C));
    RealMatrix Ba = MatrixUtils.createRealMatrix(B);
    System.out.println("Ba = " + Ba.toString());
    float[] v1 = { 1, 2, 3 };
    float[] v2 = { 4, 5, 6, 7 };
    float[] v3 = { 8 };
    float[] v123 = concatenate(v1, v2, v3);
    System.out.println("v123 = \n" + toString(v123));
    System.out.println("mind1 = " + Matrix.min(new double[] { -20, 30, 60, -40, 0 }));
    System.out.println("maxd2 = " + Matrix.max(new double[] { -20, 30, 60, -40, 0 }));
    System.out.println("minf1 = " + Matrix.min(new float[] { -20, 30, 60, -40, 0 }));
    System.out.println("maxf2 = " + Matrix.max(new float[] { -20, 30, 60, -40, 0 }));
}

Example 28 with Sum

use of org.apache.commons.math3.stat.descriptive.summary.Sum in project tetrad by cmu-phil.

the class DataUtils method cov.

public static TetradMatrix cov(TetradMatrix data) {
    for (int j = 0; j < data.columns(); j++) {
        double sum = 0.0;
        for (int i = 0; i < data.rows(); i++) {
            sum += data.get(i, j);
        }
        double mean = sum / data.rows();
        for (int i = 0; i < data.rows(); i++) {
            data.set(i, j, data.get(i, j) - mean);
        }
    }
    RealMatrix q = data.getRealMatrix();
    RealMatrix q1 = MatrixUtils.transposeWithoutCopy(q);
    RealMatrix q2 = times(q1, q);
    TetradMatrix prod = new TetradMatrix(q2, q.getColumnDimension(), q.getColumnDimension());
    double factor = 1.0 / (data.rows() - 1);
    for (int i = 0; i < prod.rows(); i++) {
        for (int j = 0; j < prod.columns(); j++) {
            prod.set(i, j, prod.get(i, j) * factor);
        }
    }
    return prod;
}

Example 29 with Sum

use of org.apache.commons.math3.stat.descriptive.summary.Sum in project narchy by automenta.

the class MyCMAESOptimizer method initializeCMA.

/**
 * Initialization of the dynamic search parameters
 *
 * @param guess Initial guess for the arguments of the fitness function.
 */
private void initializeCMA(double[] guess) {
    if (lambda <= 0) {
        throw new NotStrictlyPositiveException(lambda);
    }
    // initialize sigma
    final double[][] sigmaArray = new double[guess.length][1];
    for (int i = 0; i < guess.length; i++) {
        sigmaArray[i][0] = inputSigma[i];
    }
    final RealMatrix insigma = new Array2DRowRealMatrix(sigmaArray, false);
    // overall standard deviation
    sigma = max(insigma);
    // initialize termination criteria
    stopTolUpX = oNEtHOUSAND * max(insigma);
    stopTolX = epsilonWTF11 * max(insigma);
    this.stopTolFun = EPSILON_WTF12;
    this.stopTolHistFun = epsilonwtf13;
    // initialize selection strategy parameters
    // number of parents/points for recombination
    mu = lambda / 2;
    /* log(mu + 0.5), stored for efficiency. */
    double logMu2 = Math.log(mu + 0.5);
    weights = log(sequence(1, mu, 1)).scalarMultiply(-1).scalarAdd(logMu2);
    double sumw = 0;
    double sumwq = 0;
    for (int i = 0; i < mu; i++) {
        double w = weights.getEntry(i, 0);
        sumw += w;
        sumwq += w * w;
    }
    weights = weights.scalarMultiply(1 / sumw);
    // variance-effectiveness of sum w_i x_i
    mueff = sumw * sumw / sumwq;
    // initialize dynamic strategy parameters and constants
    cc = (4 + mueff / dimension) / (dimension + 4 + 2 * mueff / dimension);
    cs = (mueff + 2) / (dimension + mueff + 3.0);
    damps = (1 + 2 * Math.max(0, Math.sqrt((mueff - 1) / (dimension + 1)) - 1)) * Math.max(0.3, 1 - dimension / (epsilon6WTF + maxIterations)) + // minor increment
    cs;
    ccov1 = 2 / ((dimension + 1.3) * (dimension + 1.3) + mueff);
    ccovmu = Math.min(1 - ccov1, 2 * (mueff - 2 + 1 / mueff) / ((dimension + 2) * (dimension + 2) + mueff));
    ccov1Sep = Math.min(1, ccov1 * (dimension + 1.5) / 3);
    ccovmuSep = Math.min(1 - ccov1, ccovmu * (dimension + 1.5) / 3);
    chiN = Math.sqrt(dimension) * (1 - 1 / (4.0 * dimension) + 1 / (21.0 * dimension * dimension));
    // intialize CMA internal values - updated each generation
    // objective variables
    xmean = MatrixUtils.createColumnRealMatrix(guess);
    diagD = insigma.scalarMultiply(1 / sigma);
    diagC = square(diagD);
    // evolution paths for C and sigma
    pc = zeros(dimension, 1);
    // B defines the coordinate system
    ps = zeros(dimension, 1);
    normps = ps.getFrobeniusNorm();
    B = eye(dimension, dimension);
    // diagonal D defines the scaling
    D = ones(dimension, 1);
    BD = times(B, repmat(diagD.transpose(), dimension, 1));
    // covariance
    C = B.multiply(diag(square(D)).multiply(B.transpose()));
    /* Size of history queue of best values. */
    int historySize = 10 + (int) (3 * 10 * dimension / (double) lambda);
    // history of fitness values
    fitnessHistory = new double[historySize];
    for (int i = 0; i < historySize; i++) {
        fitnessHistory[i] = Double.POSITIVE_INFINITY;
    }
}

Example 30 with Sum

use of org.apache.commons.math3.stat.descriptive.summary.Sum in project tetrad by cmu-phil.

the class Ling method pruneEdgesByResampling.

// ==============================PRIVATE METHODS====================//
/**
 * This is the method used in Patrik's code.
 */
public TetradMatrix pruneEdgesByResampling(TetradMatrix data) {
    TetradMatrix X = new TetradMatrix(data.transpose().toArray());
    int npieces = 10;
    int cols = X.columns();
    int rows = X.rows();
    int piecesize = (int) Math.floor(cols / npieces);
    List<TetradMatrix> bpieces = new ArrayList<>();
    List<TetradVector> diststdpieces = new ArrayList<>();
    List<TetradVector> cpieces = new ArrayList<>();
    for (int p = 0; p < npieces; p++) {
        // % Select subset of data, and permute the variables to the causal order
        // Xp = X(k,((p-1)*piecesize+1):(p*piecesize));
        int p0 = (p) * piecesize;
        int p1 = (p + 1) * piecesize - 1;
        int[] range = range(p0, p1);
        TetradMatrix Xp = X;
        // % Remember to subract out the mean
        // Xpm = mean(Xp,2);
        // Xp = Xp - Xpm*ones(1,size(Xp,2));
        // 
        // % Calculate covariance matrix
        // cov = (Xp*Xp')/size(Xp,2);
        TetradVector Xpm = new TetradVector(rows);
        for (int i = 0; i < rows; i++) {
            double sum = 0.0;
            for (int j = 0; j < Xp.columns(); j++) {
                sum += Xp.get(i, j);
            }
            Xpm.set(i, sum / Xp.columns());
        }
        for (int i = 0; i < rows; i++) {
            for (int j = 0; j < Xp.columns(); j++) {
                Xp.set(i, j, Xp.get(i, j) - Xpm.get(i));
            }
        }
        TetradMatrix Xpt = Xp.transpose();
        TetradMatrix cov = Xp.times(Xpt);
        for (int i = 0; i < cov.rows(); i++) {
            for (int j = 0; j < cov.columns(); j++) {
                cov.set(i, j, cov.get(i, j) / Xp.columns());
            }
        }
        // % Do QL decomposition on the inverse square root of cov
        // [Q,L] = tridecomp(cov^(-0.5),'ql');
        boolean posDef = LingUtils.isPositiveDefinite(cov);
        if (!posDef) {
            System.out.println("Covariance matrix is not positive definite.");
        }
        TetradMatrix sqrt = cov.sqrt();
        ;
        TetradMatrix I = TetradMatrix.identity(rows);
        TetradMatrix AI = I.copy();
        TetradMatrix invSqrt = sqrt.inverse();
        QRDecomposition qr = new QRDecomposition(invSqrt.getRealMatrix());
        RealMatrix r = qr.getR();
        // % The estimated disturbance-stds are one over the abs of the diag of L
        // newestdisturbancestd = 1./diag(abs(L));
        TetradVector newestdisturbancestd = new TetradVector(rows);
        for (int t = 0; t < rows; t++) {
            newestdisturbancestd.set(t, 1.0 / Math.abs(r.getEntry(t, t)));
        }
        // 
        for (int s = 0; s < rows; s++) {
            for (int t = 0; t < min(s, cols); t++) {
                r.setEntry(s, t, r.getEntry(s, t) / r.getEntry(s, s));
            }
        }
        // % Calculate corresponding B
        // bnewest = eye(dims)-L;
        TetradMatrix bnewest = TetradMatrix.identity(rows);
        bnewest = bnewest.minus(new TetradMatrix(r));
        TetradVector cnewest = new TetradMatrix(r).times(Xpm);
        bpieces.add(bnewest);
        diststdpieces.add(newestdisturbancestd);
        cpieces.add(cnewest);
    }
    // 
    // for i=1:dims,
    // for j=1:dims,
    // 
    // themean = mean(Bpieces(i,j,:));
    // thestd = std(Bpieces(i,j,:));
    // if abs(themean)<prunefactor*thestd,
    // Bfinal(i,j) = 0;
    // else
    // Bfinal(i,j) = themean;
    // end
    // 
    // end
    // end
    TetradMatrix means = new TetradMatrix(rows, rows);
    TetradMatrix stds = new TetradMatrix(rows, rows);
    TetradMatrix BFinal = new TetradMatrix(rows, rows);
    for (int i = 0; i < rows; i++) {
        for (int j = 0; j < rows; j++) {
            double sum = 0.0;
            for (int y = 0; y < npieces; y++) {
                sum += bpieces.get(y).get(i, j);
            }
            double themean = sum / (npieces);
            double sumVar = 0.0;
            for (int y = 0; y < npieces; y++) {
                sumVar += Math.pow((bpieces.get(y).get(i, j)) - themean, 2);
            }
            double thestd = Math.sqrt(sumVar / (npieces));
            means.set(i, j, themean);
            stds.set(i, j, thestd);
            if (Math.abs(themean) < threshold * thestd) {
                // getPruneFactor() * thestd) {
                BFinal.set(i, j, 0);
            } else {
                BFinal.set(i, j, themean);
            }
        }
    }
    return BFinal;
}