Examples with Dim org.eclipse.test.internal.performance.data.Dim used on opensource projects

Example 6 with Dim

use of org.eclipse.test.internal.performance.data.Dim in project eclipse.platform.releng by eclipse.

the class Evaluator method evaluate.

@Override
public void evaluate(PerformanceMeter performanceMeter) throws RuntimeException {
    if (fCheckers == null)
        // nothing to do
        return;
    // get reference build tag
    Variations refKeys = PerformanceTestPlugin.getAssertAgainst();
    String assertKey = System.getProperty(PerformanceTestPlugin.ECLIPSE_PERF_ASSERTAGAINST);
    if (refKeys == null) {
        // $NON-NLS-1$ //$NON-NLS-2$
        PerformanceTestPlugin.logWarning("refkeys was null. " + PerformanceTestPlugin.ECLIPSE_PERF_ASSERTAGAINST + " was " + assertKey);
        // nothing to do
        return;
    }
    // else
    // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
    PerformanceTestPlugin.logInfo("refkeys was: " + refKeys.toString() + " \n\t based on " + PerformanceTestPlugin.ECLIPSE_PERF_ASSERTAGAINST + " being set to " + assertKey);
    if (!(performanceMeter instanceof InternalPerformanceMeter))
        // we cannot handle this.
        return;
    InternalPerformanceMeter ipm = (InternalPerformanceMeter) performanceMeter;
    Sample session = ipm.getSample();
    // $NON-NLS-1$
    Assert.assertTrue("metering session is null", session != null);
    String scenarioName = session.getScenarioID();
    // determine all dimensions we need
    HashSet<Dim> allDimensions = new HashSet<>();
    for (int i = 0; i < fCheckers.length; i++) {
        AssertChecker chk = fCheckers[i];
        Dim[] dims = chk.getDimensions();
        for (int j = 0; j < dims.length; j++) allDimensions.add(dims[j]);
    }
    // get data for this session
    DataPoint[] sessionDatapoints;
    Variations config = PerformanceTestPlugin.getVariations();
    if (config != null)
        sessionDatapoints = DB.queryDataPoints(config, scenarioName, allDimensions);
    else
        sessionDatapoints = session.getDataPoints();
    if (sessionDatapoints == null || sessionDatapoints.length == 0) {
        // $NON-NLS-1$ //$NON-NLS-2$
        PerformanceTestPlugin.logWarning("no session data named '" + config + "' found");
        return;
    }
    // get reference data
    DataPoint[] datapoints = DB.queryDataPoints(refKeys, scenarioName, allDimensions);
    if (datapoints == null || datapoints.length == 0) {
        // $NON-NLS-1$ //$NON-NLS-2$
        PerformanceTestPlugin.logWarning("no reference data named '" + refKeys + "' found");
        return;
    }
    // calculate the average
    StatisticsSession referenceStats = new StatisticsSession(datapoints);
    StatisticsSession measuredStats = new StatisticsSession(sessionDatapoints);
    // $NON-NLS-1$ //$NON-NLS-2$
    StringBuffer failMesg = new StringBuffer("Performance criteria not met when compared to '" + refKeys + "':");
    boolean pass = true;
    for (int i = 0; i < fCheckers.length; i++) {
        AssertChecker chk = fCheckers[i];
        pass &= chk.test(referenceStats, measuredStats, failMesg);
    }
    if (!pass) {
        if (config != null)
            DB.markAsFailed(config, session, failMesg.toString());
    // else
    // Assert.assertTrue(failMesg.toString(), false);
    }
}

Example 7 with Dim

use of org.eclipse.test.internal.performance.data.Dim in project eclipse.platform.releng by eclipse.

the class RelativeBandChecker method test.

@Override
public boolean test(StatisticsSession reference, StatisticsSession measured, StringBuffer message) {
    Dim dimension = getDimension();
    if (!measured.contains(dimension)) {
        // $NON-NLS-1$
        PerformanceTestPlugin.logWarning("collected data provides no dimension '" + dimension.getName() + '\'');
        return true;
    }
    if (!reference.contains(dimension)) {
        // $NON-NLS-1$
        PerformanceTestPlugin.logWarning("reference data provides no dimension '" + dimension.getName() + '\'');
        return true;
    }
    double actual = measured.getAverage(dimension);
    double test = reference.getAverage(dimension);
    if (test < 0.001 && test > -0.001) {
        // we don't fail for reference value of zero
        // $NON-NLS-1$ //$NON-NLS-2$
        PerformanceTestPlugin.logWarning("ref value for '" + dimension.getName() + "' is too small");
        return true;
    }
    if (actual < 0) {
        // we don't fail for negative values
        // $NON-NLS-1$ //$NON-NLS-2$
        PerformanceTestPlugin.logWarning("actual value for '" + dimension.getName() + "' is negative");
        return true;
    }
    if (actual > fUpperBand * test || actual < fLowerBand * test) {
        message.append('\n' + dimension.getName() + ": " + dimension.getDisplayValue(actual) + " is not within [" + Math.round(fLowerBand * 100) + "%, " + Math.round(fUpperBand * 100) + "%] of " + // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
        dimension.getDisplayValue(test));
        return false;
    }
    return true;
}

Example 8 with Dim

use of org.eclipse.test.internal.performance.data.Dim in project eclipse.platform.releng by eclipse.

the class InternalPerformanceMeter method printSample.

private void printSample(PrintStream ps, Sample sample) {
    // $NON-NLS-1$ //$NON-NLS-2$
    ps.print("Scenario '" + getScenarioName() + "' ");
    DataPoint[] dataPoints = sample.getDataPoints();
    if (dataPoints.length > 0) {
        StatisticsSession s = new StatisticsSession(dataPoints);
        Dim[] dimensions = dataPoints[0].getDimensions();
        Arrays.sort(dimensions, new DimensionComparator());
        if (dimensions.length > 0) {
            List<Dim> badDimensions = new ArrayList<>();
            long n = s.getCount(dimensions[0]);
            // $NON-NLS-1$
            MessageFormat format = new MessageFormat("({0,number,percent} in [{1}, {2}])");
            // $NON-NLS-1$
            String spaces = "                                                                                                       ";
            // $NON-NLS-1$ //$NON-NLS-2$
            ps.println("(average over " + n + " samples):");
            for (Dim dimension : dimensions) {
                double mean = s.getAverage(dimension);
                // $NON-NLS-1$ //$NON-NLS-2$
                String nameString = "  " + dimension.getName() + ":";
                String meanString = dimension.getDisplayValue(mean);
                int align = firstNonDigit(meanString);
                int endIndex = 30 - align - nameString.length();
                if (endIndex > 0)
                    meanString = spaces.substring(0, endIndex) + meanString;
                align = nameString.length() + meanString.length();
                Percentile percentile = StatisticsUtil.T95;
                double[] confidenceInterval = s.getConfidenceInterval(dimension, percentile);
                StringBuffer printBuffer;
                if (n <= 2) {
                    // $NON-NLS-1$
                    printBuffer = new StringBuffer(" (no confidence)");
                } else {
                    printBuffer = new StringBuffer();
                    int ns = align;
                    while (ns++ < 40) printBuffer.append(' ');
                    printBuffer.append(format.format(new Object[] { Double.valueOf(percentile.inside()), dimension.getDisplayValue(confidenceInterval[0]), dimension.getDisplayValue(confidenceInterval[1]) }));
                }
                align += printBuffer.length();
                try {
                    while (align++ < 70) printBuffer.append(' ');
                    printBuffer.append(checkSampleSize(s, sample, dimension));
                } catch (CoreException x) {
                    badDimensions.add(dimension);
                    continue;
                }
                ps.print(nameString);
                ps.print(meanString);
                ps.println(printBuffer);
            }
            if (!badDimensions.isEmpty()) {
                // $NON-NLS-1$
                ps.print("  Dimensions with unusable statistical properties: ");
                for (Iterator<Dim> iter = badDimensions.iterator(); iter.hasNext(); ) {
                    Dim dimension = iter.next();
                    ps.print(dimension.getName());
                    if (iter.hasNext())
                        // $NON-NLS-1$
                        ps.print(", ");
                }
                ps.println();
            }
        }
    }
    ps.println();
}

Example 9 with Dim

use of org.eclipse.test.internal.performance.data.Dim in project eclipse.platform.releng by eclipse.

the class Scenario method dump.

public void dump(PrintStream ps, String key) {
    // $NON-NLS-1$
    ps.println("Scenario: " + getScenarioName());
    Report r = new Report(2);
    String[] timeSeriesLabels = getTimeSeriesLabels();
    // $NON-NLS-1$
    r.addCell(key + ":");
    for (String timeSeriesLabel : timeSeriesLabels) r.addCellRight(timeSeriesLabel);
    r.nextRow();
    Dim[] dimensions = getDimensions();
    for (Dim dim : dimensions) {
        r.addCell(dim.getName() + ':');
        TimeSeries ts = getTimeSeries(dim);
        int n = ts.getLength();
        for (int j = 0; j < n; j++) {
            // $NON-NLS-1$
            String stddev = "";
            double stddev2 = ts.getStddev(j);
            if (stddev2 != 0.0)
                // $NON-NLS-1$ //$NON-NLS-2$
                stddev = " [" + dim.getDisplayValue(stddev2) + "]";
            r.addCellRight(dim.getDisplayValue(ts.getValue(j)) + stddev);
        }
        r.nextRow();
    }
    r.print(ps);
    ps.println();
}

Example 10 with Dim

use of org.eclipse.test.internal.performance.data.Dim in project eclipse.platform.releng by eclipse.

the class InternalPerformanceMeter method printSampleCSV.

void printSampleCSV(PrintStream ps, Sample sample) {
    final char SEPARATOR = '\t';
    DataPoint[] dataPoints = sample.getDataPoints();
    if (dataPoints.length > 0) {
        Dim[] dimensions = dataPoints[0].getDimensions();
        Arrays.sort(dimensions, new DimensionComparator());
        if (dimensions.length > 0) {
            /* print dimensions */
            for (Dim dimension : dimensions) {
                ps.print(dimension.getName());
                ps.print(SEPARATOR);
            }
            // $NON-NLS-1$
            ps.println("scenario");
            for (int p = 0; p < dataPoints.length - 1; p += 2) {
                DataPoint before = dataPoints[p];
                DataPoint after = dataPoints[p + 1];
                for (Dim dimension : dimensions) {
                    long valBefore = before.getScalar(dimension).getMagnitude();
                    long valAfter = after.getScalar(dimension).getMagnitude();
                    ps.print(valAfter - valBefore);
                    ps.print(SEPARATOR);
                }
                ps.print(sample.getShortname() != null ? sample.getShortname() : sample.getScenarioID());
                ps.println();
            }
            ps.println();
        }
    }
}