Examples with DescriptiveStatistics org.apache.commons.math3.stat.descriptive.DescriptiveStatistics used on opensource projects

Example 16 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project lightning by automatictester.

the class ServerSideTest method execute.

@Override
public void execute(ArrayList<String[]> originalDataEntries) {
    try {
        PerfMonDataEntries dataEntries = filterDataEntries((PerfMonDataEntries) originalDataEntries);
        dataEntriesCount = dataEntries.getDataEntriesCount();
        DescriptiveStatistics ds = new DescriptiveStatistics();
        for (String[] transaction : dataEntries) {
            String elapsed = transaction[1];
            ds.addValue(Double.parseDouble(elapsed));
        }
        actualResult = (int) ds.getMean();
        actualResultDescription = String.format(ACTUAL_RESULT_MESSAGE, actualResult);
        if (subtype.equals(ServerSideTestType.GREATER_THAN)) {
            expectedResultDescription = String.format(expectedResultMessage, metricValueA);
            if (actualResult > metricValueA) {
                result = TestResult.PASS;
            } else {
                result = TestResult.FAIL;
            }
        } else if (subtype.equals(ServerSideTestType.LESS_THAN)) {
            expectedResultDescription = String.format(expectedResultMessage, metricValueA);
            if (actualResult < metricValueA) {
                result = TestResult.PASS;
            } else {
                result = TestResult.FAIL;
            }
        } else if (subtype.equals(ServerSideTestType.BETWEEN)) {
            expectedResultDescription = String.format(expectedResultMessage, metricValueA, metricValueB);
            if ((actualResult > metricValueA) && (actualResult < metricValueB)) {
                result = TestResult.PASS;
            } else {
                result = TestResult.FAIL;
            }
        }
    } catch (Exception e) {
        result = TestResult.ERROR;
        actualResultDescription = e.getMessage();
    }
}

Example 17 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project lightning by automatictester.

the class RespTimeAvgTest method execute.

@Override
public void execute(ArrayList<String[]> originalJMeterTransactions) {
    try {
        JMeterTransactions transactions = filterTransactions((JMeterTransactions) originalJMeterTransactions);
        transactionCount = transactions.getTransactionCount();
        DescriptiveStatistics ds = new DescriptiveStatistics();
        for (String[] transaction : transactions) {
            String elapsed = transaction[1];
            ds.addValue(Double.parseDouble(elapsed));
        }
        longestTransactions = transactions.getLongestTransactions();
        actualResult = (int) ds.getMean();
        actualResultDescription = String.format(ACTUAL_RESULT_MESSAGE, actualResult);
        if (actualResult > maxAvgRespTime) {
            result = TestResult.FAIL;
        } else {
            result = TestResult.PASS;
        }
    } catch (Exception e) {
        result = TestResult.ERROR;
        actualResultDescription = e.getMessage();
    }
}

Example 18 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project presto by prestodb.

the class AbstractTestApproximateCountDistinct method testMultiplePositions.

@Test(dataProvider = "provideStandardErrors")
public void testMultiplePositions(double maxStandardError) {
    DescriptiveStatistics stats = new DescriptiveStatistics();
    for (int i = 0; i < 500; ++i) {
        int uniques = ThreadLocalRandom.current().nextInt(getUniqueValuesCount()) + 1;
        List<Object> values = createRandomSample(uniques, (int) (uniques * 1.5));
        long actual = estimateGroupByCount(values, maxStandardError);
        double error = (actual - uniques) * 1.0 / uniques;
        stats.addValue(error);
    }
    assertLessThan(stats.getMean(), 1.0e-2);
    assertLessThan(stats.getStandardDeviation(), 1.0e-2 + maxStandardError);
}

Example 19 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project cassandra by apache.

the class DriverBurnTest method perfTest.

public void perfTest(SizeCaps requestCaps, SizeCaps responseCaps, Cluster.Builder builder, ProtocolVersion version) throws Throwable {
    SimpleStatement request = generateQueryStatement(0, requestCaps);
    ResultMessage.Rows response = generateRows(0, responseCaps);
    QueryMessage requestMessage = generateQueryMessage(0, requestCaps, version);
    Envelope message = requestMessage.encode(version);
    int requestSize = message.body.readableBytes();
    message.release();
    message = response.encode(version);
    int responseSize = message.body.readableBytes();
    message.release();
    Message.Type.QUERY.unsafeSetCodec(new Message.Codec<QueryMessage>() {

        public QueryMessage decode(ByteBuf body, ProtocolVersion version) {
            QueryMessage queryMessage = QueryMessage.codec.decode(body, version);
            return new QueryMessage(queryMessage.query, queryMessage.options) {

                protected Message.Response execute(QueryState state, long queryStartNanoTime, boolean traceRequest) {
                    try {
                        // unused
                        int idx = Integer.parseInt(queryMessage.query);
                        return generateRows(idx, responseCaps);
                    } catch (NumberFormatException e) {
                        // for the requests driver issues under the hood
                        return super.execute(state, queryStartNanoTime, traceRequest);
                    }
                }
            };
        }

        public void encode(QueryMessage queryMessage, ByteBuf dest, ProtocolVersion version) {
            QueryMessage.codec.encode(queryMessage, dest, version);
        }

        public int encodedSize(QueryMessage queryMessage, ProtocolVersion version) {
            return QueryMessage.codec.encodedSize(queryMessage, version);
        }
    });
    int threads = 10;
    int perThread = 30;
    ExecutorService executor = Executors.newFixedThreadPool(threads + 10);
    AtomicReference<Throwable> error = new AtomicReference<>();
    CountDownLatch signal = new CountDownLatch(1);
    AtomicBoolean measure = new AtomicBoolean(false);
    DescriptiveStatistics stats = new DescriptiveStatistics();
    Lock lock = new ReentrantLock();
    for (int t = 0; t < threads; t++) {
        executor.execute(() -> {
            try (Cluster driver = builder.build();
                Session session = driver.connect()) {
                while (!executor.isShutdown() && error.get() == null) {
                    Map<Integer, ResultSetFuture> futures = new HashMap<>();
                    for (int j = 0; j < perThread; j++) {
                        long startNanos = nanoTime();
                        ResultSetFuture future = session.executeAsync(request);
                        future.addListener(() -> {
                            long diff = nanoTime() - startNanos;
                            if (measure.get()) {
                                lock.lock();
                                try {
                                    stats.addValue(TimeUnit.MICROSECONDS.toMillis(diff));
                                } finally {
                                    lock.unlock();
                                }
                            }
                        }, executor);
                        futures.put(j, future);
                    }
                    for (Map.Entry<Integer, ResultSetFuture> e : futures.entrySet()) {
                        Assert.assertEquals(response.result.size(), e.getValue().get().all().size());
                    }
                }
            } catch (Throwable e) {
                e.printStackTrace();
                error.set(e);
                signal.countDown();
            }
        });
    }
    Assert.assertFalse(signal.await(30, TimeUnit.SECONDS));
    measure.set(true);
    Assert.assertFalse(signal.await(60, TimeUnit.SECONDS));
    executor.shutdown();
    executor.awaitTermination(10, TimeUnit.SECONDS);
    System.out.println("requestSize = " + requestSize);
    System.out.println("responseSize = " + responseSize);
    System.out.println("Mean:     " + stats.getMean());
    System.out.println("Variance: " + stats.getVariance());
    System.out.println("Median:   " + stats.getPercentile(0.5));
    System.out.println("90p:      " + stats.getPercentile(0.90));
    System.out.println("95p:      " + stats.getPercentile(0.95));
    System.out.println("99p:      " + stats.getPercentile(0.99));
}

Example 20 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project pyramid by cheng-li.

the class GMMDemo method main.

public static void main(String[] args) throws Exception {
    List<String> lines = FileUtils.readLines(new File("/Users/chengli/Dropbox/Shared/CS6220DM/2_cluster_EM_mixt/HW2/mnist_features.txt"));
    Collections.shuffle(lines);
    int dim = lines.get(0).split(" ").length;
    int rows = 100;
    RealMatrix data = new Array2DRowRealMatrix(rows, dim);
    for (int i = 0; i < rows; i++) {
        String[] split = lines.get(i).split(" ");
        for (int j = 0; j < dim; j++) {
            data.setEntry(i, j, Double.parseDouble(split[j]) + Math.random());
        }
    }
    double[] mins = new double[data.getColumnDimension()];
    double[] maxs = new double[data.getColumnDimension()];
    double[] vars = new double[data.getColumnDimension()];
    for (int j = 0; j < data.getColumnDimension(); j++) {
        RealVector column = data.getColumnVector(j);
        mins[j] = column.getMinValue();
        maxs[j] = column.getMaxValue();
        DescriptiveStatistics stats = new DescriptiveStatistics(column.toArray());
        vars[j] = stats.getVariance();
    }
// DataSet dataSet = DataSetBuilder.getBuilder()
// .numDataPoints(rows)
// .numFeatures(data.getColumnDimension())
// .build();
// for (int i=0;i<dataSet.getNumDataPoints();i++){
// for (int j=0;j<dataSet.getNumFeatures();j++){
// if (data.getEntry(i,j)>255.0/2){
// dataSet.setFeatureValue(i,j,1);
// } else {
// dataSet.setFeatureValue(i,j,0);
// }
// 
// }
// }
// 
// int numComponents = 10;
// 
// 
// BM bm = BMSelector.select(dataSet, numComponents, 100);
// System.out.println(Arrays.toString(bm.getMixtureCoefficients()));
// StringBuilder stringBuilder = new StringBuilder();
// for (int k=0;k<numComponents;k++){
// for (int d=0;d<dataSet.getNumFeatures();d++){
// stringBuilder.append(bm.getDistributions()[k][d].getP());
// if (d!=dataSet.getNumFeatures()-1){
// stringBuilder.append(",");
// }
// }
// stringBuilder.append("\n");
// }
// FileUtils.writeStringToFile(new File("/Users/chengli/tmp/gmm/bm"),stringBuilder.toString());
// BMTrainer bmTrainer = BMSelector.selectTrainer(dataSet,numComponents,50);
// 
// GMM gmm = new GMM(dim,numComponents, data);
// 
// GMMTrainer trainer = new GMMTrainer(data, gmm);
// 
// trainer.setGammas(bmTrainer.getGammas());
// trainer.mStep();
// 
// for (int i=1;i<=5;i++){
// System.out.println("iteration = "+i);
// trainer.iterate();
// double logLikelihood = IntStream.range(0,rows).parallel()
// .mapToDouble(j->gmm.logDensity(data.getRowVector(j))).sum();
// System.out.println("log likelihood = "+logLikelihood);
// Serialization.serialize(gmm, "/Users/chengli/tmp/gmm/model_iter_"+i);
// for (int k=0;k<gmm.getNumComponents();k++){
// FileUtils.writeStringToFile(new File("/Users/chengli/tmp/gmm/mean_iter_"+i+"_component_"+(k+1)),
// gmm.getGaussianDistributions()[k].getMean().toString().replace("{","")
// .replace("}","").replace(";",","));
// }
// }
// 
// FileUtils.writeStringToFile(new File("/Users/chengli/tmp/gmm/modeltext"), gmm.toString());
// GMM gmm = (GMM) Serialization.deserialize("/Users/chengli/tmp/gmm/model_3");
}