Examples with Variations org.eclipse.test.internal.performance.db.Variations used on opensource projects

Example 1 with Variations

use of org.eclipse.test.internal.performance.db.Variations in project eclipse.platform.releng by eclipse.

the class VariationsTests method testVariations.

public void testVariations() {
    Variations v1 = new Variations();
    // $NON-NLS-1$ //$NON-NLS-2$
    v1.put("k1", "foo");
    // $NON-NLS-1$
    assertEquals("|k1=foo|", v1.toExactMatchString());
    // $NON-NLS-1$
    assertEquals("%|k1=foo|%", v1.toQueryPattern());
    Variations v2 = new Variations();
    // $NON-NLS-1$ //$NON-NLS-2$
    v2.put("k1", "foo");
    // $NON-NLS-1$ //$NON-NLS-2$
    v2.put("k2", "bar");
    // $NON-NLS-1$
    assertEquals("|k1=foo||k2=bar|", v2.toExactMatchString());
    // $NON-NLS-1$
    assertEquals("%|k1=foo|%|k2=bar|%", v2.toQueryPattern());
    Variations v3 = new Variations();
    // $NON-NLS-1$ //$NON-NLS-2$
    v3.put("k1", "foo");
    // $NON-NLS-1$ //$NON-NLS-2$
    v3.put("k2", "bar");
    // $NON-NLS-1$ //$NON-NLS-2$
    v3.put("k3", "xyz");
    // $NON-NLS-1$
    assertEquals("|k1=foo||k2=bar||k3=xyz|", v3.toExactMatchString());
    // $NON-NLS-1$
    assertEquals("%|k1=foo|%|k2=bar|%|k3=xyz|%", v3.toQueryPattern());
}

Example 2 with Variations

use of org.eclipse.test.internal.performance.db.Variations in project eclipse.platform.releng by eclipse.

the class Evaluator method evaluate.

@Override
public void evaluate(PerformanceMeter performanceMeter) throws RuntimeException {
    if (fCheckers == null)
        // nothing to do
        return;
    // get reference build tag
    Variations refKeys = PerformanceTestPlugin.getAssertAgainst();
    String assertKey = System.getProperty(PerformanceTestPlugin.ECLIPSE_PERF_ASSERTAGAINST);
    if (refKeys == null) {
        // $NON-NLS-1$ //$NON-NLS-2$
        PerformanceTestPlugin.logWarning("refkeys was null. " + PerformanceTestPlugin.ECLIPSE_PERF_ASSERTAGAINST + " was " + assertKey);
        // nothing to do
        return;
    }
    // else
    // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
    PerformanceTestPlugin.logInfo("refkeys was: " + refKeys.toString() + " \n\t based on " + PerformanceTestPlugin.ECLIPSE_PERF_ASSERTAGAINST + " being set to " + assertKey);
    if (!(performanceMeter instanceof InternalPerformanceMeter))
        // we cannot handle this.
        return;
    InternalPerformanceMeter ipm = (InternalPerformanceMeter) performanceMeter;
    Sample session = ipm.getSample();
    // $NON-NLS-1$
    Assert.assertTrue("metering session is null", session != null);
    String scenarioName = session.getScenarioID();
    // determine all dimensions we need
    HashSet<Dim> allDimensions = new HashSet<>();
    for (int i = 0; i < fCheckers.length; i++) {
        AssertChecker chk = fCheckers[i];
        Dim[] dims = chk.getDimensions();
        for (int j = 0; j < dims.length; j++) allDimensions.add(dims[j]);
    }
    // get data for this session
    DataPoint[] sessionDatapoints;
    Variations config = PerformanceTestPlugin.getVariations();
    if (config != null)
        sessionDatapoints = DB.queryDataPoints(config, scenarioName, allDimensions);
    else
        sessionDatapoints = session.getDataPoints();
    if (sessionDatapoints == null || sessionDatapoints.length == 0) {
        // $NON-NLS-1$ //$NON-NLS-2$
        PerformanceTestPlugin.logWarning("no session data named '" + config + "' found");
        return;
    }
    // get reference data
    DataPoint[] datapoints = DB.queryDataPoints(refKeys, scenarioName, allDimensions);
    if (datapoints == null || datapoints.length == 0) {
        // $NON-NLS-1$ //$NON-NLS-2$
        PerformanceTestPlugin.logWarning("no reference data named '" + refKeys + "' found");
        return;
    }
    // calculate the average
    StatisticsSession referenceStats = new StatisticsSession(datapoints);
    StatisticsSession measuredStats = new StatisticsSession(sessionDatapoints);
    // $NON-NLS-1$ //$NON-NLS-2$
    StringBuffer failMesg = new StringBuffer("Performance criteria not met when compared to '" + refKeys + "':");
    boolean pass = true;
    for (int i = 0; i < fCheckers.length; i++) {
        AssertChecker chk = fCheckers[i];
        pass &= chk.test(referenceStats, measuredStats, failMesg);
    }
    if (!pass) {
        if (config != null)
            DB.markAsFailed(config, session, failMesg.toString());
    // else
    // Assert.assertTrue(failMesg.toString(), false);
    }
}

Example 3 with Variations

use of org.eclipse.test.internal.performance.db.Variations in project eclipse.platform.releng by eclipse.

the class DBTests method testAssertPerformance.

public void testAssertPerformance() throws SQLException {
    Performance perf = Performance.getDefault();
    // set the variation for the reference data
    // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
    System.setProperty("eclipse.perf.config", CONFIG + "=test;" + BUILD + "=ref");
    // store a reference value
    TestPerformanceMeter pm1 = new TestPerformanceMeter(SCENARIO_NAME_0);
    pm1.addPair(InternalDimensions.CPU_TIME, 100, 1000);
    pm1.addPair(InternalDimensions.WORKING_SET, 1000, 2000);
    pm1.start();
    pm1.stop();
    pm1.commit();
    pm1.dispose();
    // $NON-NLS-1$
    String build = "001";
    // set the variation for the this run
    // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
    System.setProperty("eclipse.perf.config", CONFIG + "=test;" + BUILD + "=" + build);
    // assert that this run's values are compared against reference data
    // $NON-NLS-1$ //$NON-NLS-2$
    System.setProperty("eclipse.perf.assertAgainst", BUILD + "=ref");
    // store a reference value
    TestPerformanceMeter pm2 = new TestPerformanceMeter(SCENARIO_NAME_0);
    pm2.addPair(InternalDimensions.CPU_TIME, 100, 1100);
    pm2.addPair(InternalDimensions.WORKING_SET, 1000, 2000);
    pm2.start();
    pm2.stop();
    pm2.commit();
    boolean failed = false;
    try {
        perf.assertPerformanceInRelativeBand(pm2, InternalDimensions.CPU_TIME, -5, +5);
    } catch (AssertionFailedError e) {
        failed = true;
    }
    pm2.dispose();
    if (!failed) {
        // check in DB
        Variations v = new Variations();
        // $NON-NLS-1$
        v.put(CONFIG, "%");
        v.put(BUILD, build);
        Scenario[] scenarios = DB.queryScenarios(v, SCENARIO_NAME_0, CONFIG, null);
        if (scenarios != null && scenarios.length > 0) {
            Scenario s = scenarios[0];
            String[] failureMessages = s.getFailureMessages();
            if (failureMessages.length == 1) {
                String mesg = failureMessages[0];
                if (mesg != null && // $NON-NLS-1$
                mesg.equals("Performance criteria not met when compared to '{b=ref, c=test}':\nCPU Time: 1 s is not within [95%, 105%] of 900 ms"))
                    failed = true;
            }
        }
    }
    assertTrue(failed);
}

Example 4 with Variations

use of org.eclipse.test.internal.performance.db.Variations in project eclipse.platform.releng by eclipse.

the class PerformanceTestPlugin method getDBUser.

public static String getDBUser() {
    String dbloc = System.getProperty(ECLIPSE_PERF_DBLOC);
    if (dbloc != null) {
        // $NON-NLS-1$ //$NON-NLS-2$
        logInfo("System property " + ECLIPSE_PERF_DBLOC + " found to equal " + dbloc + " in getDBUser");
        Variations keys = new Variations();
        keys.parsePairs(ECLIPSE_PERF_DBLOC + '=' + dbloc);
        return keys.getProperty(DB_USER, DEFAULT_DB_USER);
    }
    return DEFAULT_DB_USER;
}

Example 5 with Variations

use of org.eclipse.test.internal.performance.db.Variations in project eclipse.platform.releng by eclipse.

the class PerformanceTestPlugin method getAssertAgainst.

/*
   * -Declipse.perf.assertAgainst=<varname1>=<varval1>;<varname2>=<varval2>;...;<varnameN>=<varvalN> Returns null if assertAgainst
   * property isn't defined.
   */
public static Variations getAssertAgainst() {
    String assertKey = System.getProperty(ECLIPSE_PERF_ASSERTAGAINST);
    if (assertKey != null) {
        Variations keys = getVariations();
        if (keys == null)
            keys = new Variations();
        keys.parsePairs(assertKey);
        return keys;
    }
    return null;
}