Examples with TestExecutionResult org.jenkins.tools.test.model.TestExecutionResult used on opensource projects

Example 1 with TestExecutionResult

use of org.jenkins.tools.test.model.TestExecutionResult in project plugin-compat-tester by jenkinsci.

the class SkipUIHelperPlugins method action.

/**
 * The plugin was identified as somethig that should be skipped.
 * Create a TestExecution result preventing forward movement.
 * Also, indicates that we should skip the checkout completely.
 */
public Map<String, Object> action(Map<String, Object> moreInfo) throws Exception {
    moreInfo.put("executionResult", new TestExecutionResult(Arrays.asList("Plugin unsupported at this time, skipping")));
    moreInfo.put("runCheckout", false);
    return moreInfo;
}

Example 2 with TestExecutionResult

use of org.jenkins.tools.test.model.TestExecutionResult in project plugin-compat-tester by jenkinsci.

the class PluginCompatTester method testPlugins.

public PluginCompatReport testPlugins() throws PlexusContainerException, IOException, MavenEmbedderException {
    File war = config.getWar();
    if (war != null) {
        populateSplits(war);
    } else {
        // TODO find a way to load the local version of jenkins.war acc. to UC metadata
        splits = HISTORICAL_SPLITS;
        splitCycles = HISTORICAL_SPLIT_CYCLES;
    }
    PluginCompatTesterHooks pcth = new PluginCompatTesterHooks(config.getHookPrefixes());
    // Providing XSL Stylesheet along xml report file
    if (config.reportFile != null) {
        if (config.isProvideXslReport()) {
            File xslFilePath = PluginCompatReport.getXslFilepath(config.reportFile);
            FileUtils.copyStreamToFile(new RawInputStreamFacade(getXslTransformerResource().getInputStream()), xslFilePath);
        }
    }
    DataImporter dataImporter = null;
    if (config.getGaeBaseUrl() != null && config.getGaeSecurityToken() != null) {
        dataImporter = new DataImporter(config.getGaeBaseUrl(), config.getGaeSecurityToken());
    }
    // Determine the plugin data
    // Used to track real plugin groupIds from WARs
    HashMap<String, String> pluginGroupIds = new HashMap<String, String>();
    UpdateSite.Data data = config.getWar() == null ? extractUpdateCenterData(pluginGroupIds) : scanWAR(config.getWar(), pluginGroupIds);
    final Map<String, Plugin> pluginsToCheck;
    final List<String> pluginsToInclude = config.getIncludePlugins();
    if (data.plugins.isEmpty() && pluginsToInclude != null && !pluginsToInclude.isEmpty()) {
        // Update Center returns empty info OR the "-war" option is specified for WAR without bundled plugins
        // TODO: Ideally we should do this tweak in any case, so that we can test custom plugins with Jenkins cores before unbundling
        // But it will require us to always poll the update center...
        System.out.println("WAR file does not contain plugin info, will try to extract it from UC for included plugins");
        pluginsToCheck = new HashMap<>(pluginsToInclude.size());
        UpdateSite.Data ucData = extractUpdateCenterData(pluginGroupIds);
        for (String plugin : pluginsToInclude) {
            UpdateSite.Plugin pluginData = ucData.plugins.get(plugin);
            if (pluginData != null) {
                System.out.println("Adding " + plugin + " to the test scope");
                pluginsToCheck.put(plugin, pluginData);
            }
        }
    } else {
        pluginsToCheck = data.plugins;
    }
    if (pluginsToCheck.isEmpty()) {
        throw new IOException("List of plugins to check is empty, it is not possible to run PCT");
    }
    PluginCompatReport report = PluginCompatReport.fromXml(config.reportFile);
    SortedSet<MavenCoordinates> testedCores = config.getWar() == null ? generateCoreCoordinatesToTest(data, report) : coreVersionFromWAR(data);
    MavenRunner.Config mconfig = new MavenRunner.Config();
    mconfig.userSettingsFile = config.getM2SettingsFile();
    // TODO REMOVE
    mconfig.userProperties.put("failIfNoTests", "false");
    mconfig.userProperties.put("argLine", "-XX:MaxPermSize=128m");
    String mavenPropertiesFilePath = this.config.getMavenPropertiesFile();
    if (StringUtils.isNotBlank(mavenPropertiesFilePath)) {
        File file = new File(mavenPropertiesFilePath);
        if (file.exists()) {
            FileInputStream fileInputStream = null;
            try {
                fileInputStream = new FileInputStream(file);
                Properties properties = new Properties();
                properties.load(fileInputStream);
                for (Map.Entry<Object, Object> entry : properties.entrySet()) {
                    mconfig.userProperties.put((String) entry.getKey(), (String) entry.getValue());
                }
            } finally {
                IOUtils.closeQuietly(fileInputStream);
            }
        } else {
            System.out.println("File " + mavenPropertiesFilePath + " not exists");
        }
    }
    SCMManagerFactory.getInstance().start();
    for (MavenCoordinates coreCoordinates : testedCores) {
        System.out.println("Starting plugin tests on core coordinates : " + coreCoordinates.toString());
        for (Plugin plugin : pluginsToCheck.values()) {
            if (config.getIncludePlugins() == null || config.getIncludePlugins().contains(plugin.name.toLowerCase())) {
                PluginInfos pluginInfos = new PluginInfos(plugin.name, plugin.version, plugin.url);
                if (config.getExcludePlugins() != null && config.getExcludePlugins().contains(plugin.name.toLowerCase())) {
                    System.out.println("Plugin " + plugin.name + " is in excluded plugins => test skipped !");
                    continue;
                }
                String errorMessage = null;
                TestStatus status = null;
                MavenCoordinates actualCoreCoordinates = coreCoordinates;
                PluginRemoting remote;
                if (localCheckoutProvided() && onlyOnePluginIncluded()) {
                    // Only one plugin and checkout directory provided
                    remote = new PluginRemoting(new File(config.getLocalCheckoutDir(), "pom.xml"));
                } else if (localCheckoutProvided()) {
                    // local directory provided for more than one plugin, so each plugin is allocated in localCheckoutDir/plugin-name
                    // If there is no subdirectory for the plugin, it will be cloned from scm
                    File pomFile = new File(new File(config.getLocalCheckoutDir(), plugin.name), "pom.xml");
                    if (pomFile.exists()) {
                        remote = new PluginRemoting(pomFile);
                    } else {
                        remote = new PluginRemoting(plugin.url);
                    }
                } else {
                    // Only one plugin but checkout directory not provided or
                    // more than a plugin and no local checkout directory provided
                    remote = new PluginRemoting(plugin.url);
                }
                PomData pomData;
                try {
                    pomData = remote.retrievePomData();
                    System.out.println("detected parent POM " + pomData.parent.toGAV());
                    if ((pomData.parent.groupId.equals(PluginCompatTesterConfig.DEFAULT_PARENT_GROUP) && pomData.parent.artifactId.equals(PluginCompatTesterConfig.DEFAULT_PARENT_ARTIFACT) || pomData.parent.groupId.equals("org.jvnet.hudson.plugins")) && coreCoordinates.version.matches("1[.][0-9]+[.][0-9]+") && new VersionNumber(coreCoordinates.version).compareTo(new VersionNumber("1.485")) < 0) {
                        // TODO unless 1.480.3+
                        System.out.println("Cannot test against " + coreCoordinates.version + " due to lack of deployed POM for " + coreCoordinates.toGAV());
                        actualCoreCoordinates = new MavenCoordinates(coreCoordinates.groupId, coreCoordinates.artifactId, coreCoordinates.version.replaceFirst("[.][0-9]+$", ""));
                    }
                } catch (Throwable t) {
                    status = TestStatus.INTERNAL_ERROR;
                    errorMessage = t.getMessage();
                    pomData = null;
                }
                if (!config.isSkipTestCache() && report.isCompatTestResultAlreadyInCache(pluginInfos, actualCoreCoordinates, config.getTestCacheTimeout(), config.getCacheThresholStatus())) {
                    System.out.println("Cache activated for plugin " + pluginInfos.pluginName + " => test skipped !");
                    // Don't do anything : we are in the cached interval ! :-)
                    continue;
                }
                List<String> warningMessages = new ArrayList<String>();
                if (errorMessage == null) {
                    try {
                        TestExecutionResult result = testPluginAgainst(actualCoreCoordinates, plugin, mconfig, pomData, pluginsToCheck, pluginGroupIds, pcth);
                        // If no PomExecutionException, everything went well...
                        status = TestStatus.SUCCESS;
                        warningMessages.addAll(result.pomWarningMessages);
                    } catch (PomExecutionException e) {
                        if (!e.succeededPluginArtifactIds.contains("maven-compiler-plugin")) {
                            status = TestStatus.COMPILATION_ERROR;
                        } else if (!e.succeededPluginArtifactIds.contains("maven-surefire-plugin")) {
                            status = TestStatus.TEST_FAILURES;
                        } else {
                            // Can this really happen ???
                            status = TestStatus.SUCCESS;
                        }
                        errorMessage = e.getErrorMessage();
                        warningMessages.addAll(e.getPomWarningMessages());
                    } catch (Error e) {
                        // Rethrow the error ... something is wrong !
                        throw e;
                    } catch (Throwable t) {
                        status = TestStatus.INTERNAL_ERROR;
                        errorMessage = t.getMessage();
                    }
                }
                File buildLogFile = createBuildLogFile(config.reportFile, plugin.name, plugin.version, actualCoreCoordinates);
                String buildLogFilePath = "";
                if (buildLogFile.exists()) {
                    buildLogFilePath = createBuildLogFilePathFor(pluginInfos.pluginName, pluginInfos.pluginVersion, actualCoreCoordinates);
                }
                PluginCompatResult result = new PluginCompatResult(actualCoreCoordinates, status, errorMessage, warningMessages, buildLogFilePath);
                report.add(pluginInfos, result);
                // Adding result to GAE
                if (dataImporter != null) {
                    dataImporter.importPluginCompatResult(result, pluginInfos, config.reportFile.getParentFile());
                // TODO: import log files
                }
                if (config.reportFile != null) {
                    if (!config.reportFile.exists()) {
                        FileUtils.fileWrite(config.reportFile.getAbsolutePath(), "");
                    }
                    report.save(config.reportFile);
                }
            } else {
                System.out.println("Plugin " + plugin.name + " not in included plugins => test skipped !");
            }
        }
    }
    // Generating HTML report if needed
    if (config.reportFile != null) {
        if (config.isGenerateHtmlReport()) {
            generateHtmlReportFile();
        }
    }
    return report;
}

Example 3 with TestExecutionResult

use of org.jenkins.tools.test.model.TestExecutionResult in project plugin-compat-tester by jenkinsci.

the class PluginCompatTester method testPluginAgainst.

private TestExecutionResult testPluginAgainst(MavenCoordinates coreCoordinates, Plugin plugin, MavenRunner.Config mconfig, PomData pomData, Map<String, Plugin> otherPlugins, Map<String, String> pluginGroupIds, PluginCompatTesterHooks pcth) throws PluginSourcesUnavailableException, PomTransformationException, PomExecutionException, IOException {
    System.out.println(String.format("%n%n%n%n%n"));
    System.out.println(String.format("#############################################"));
    System.out.println(String.format("#############################################"));
    System.out.println(String.format("##%n## Starting to test plugin %s v%s%n## against %s%n##", plugin.name, plugin.version, coreCoordinates));
    System.out.println(String.format("#############################################"));
    System.out.println(String.format("#############################################"));
    System.out.println(String.format("%n%n%n%n%n"));
    File pluginCheckoutDir = new File(config.workDirectory.getAbsolutePath() + File.separator + plugin.name + File.separator);
    try {
        // Run any precheckout hooks
        Map<String, Object> beforeCheckout = new HashMap<String, Object>();
        beforeCheckout.put("pluginName", plugin.name);
        beforeCheckout.put("plugin", plugin);
        beforeCheckout.put("pomData", pomData);
        beforeCheckout.put("config", config);
        beforeCheckout.put("runCheckout", true);
        beforeCheckout = pcth.runBeforeCheckout(beforeCheckout);
        if (beforeCheckout.get("executionResult") != null) {
            // Check if the hook returned a result
            return (TestExecutionResult) beforeCheckout.get("executionResult");
        } else if ((boolean) beforeCheckout.get("runCheckout")) {
            if (beforeCheckout.get("checkoutDir") != null) {
                pluginCheckoutDir = (File) beforeCheckout.get("checkoutDir");
            }
            if (pluginCheckoutDir.exists()) {
                System.out.println("Deleting working directory " + pluginCheckoutDir.getAbsolutePath());
                FileUtils.deleteDirectory(pluginCheckoutDir);
            }
            pluginCheckoutDir.mkdir();
            System.out.println("Created plugin checkout dir : " + pluginCheckoutDir.getAbsolutePath());
            if (localCheckoutProvided()) {
                if (!onlyOnePluginIncluded()) {
                    File localCheckoutPluginDir = new File(config.getLocalCheckoutDir(), plugin.name);
                    File pomLocalCheckoutPluginDir = new File(localCheckoutPluginDir, "pom.xml");
                    if (pomLocalCheckoutPluginDir.exists()) {
                        System.out.println("Copy plugin directory from : " + localCheckoutPluginDir.getAbsolutePath());
                        FileUtils.copyDirectoryStructure(localCheckoutPluginDir, pluginCheckoutDir);
                    } else {
                        cloneFromSCM(pomData.getConnectionUrl(), plugin.name, plugin.version, pluginCheckoutDir);
                    }
                } else {
                    // TODO this fails when it encounters symlinks (e.g. work/jobs/…/builds/lastUnstableBuild),
                    // and even up-to-date versions of org.apache.commons.io.FileUtils seem to not handle links,
                    // so may need to use something like http://docs.oracle.com/javase/tutorial/displayCode.html?code=http://docs.oracle.com/javase/tutorial/essential/io/examples/Copy.java
                    System.out.println("Copy plugin directory from : " + config.getLocalCheckoutDir().getAbsolutePath());
                    FileUtils.copyDirectoryStructure(config.getLocalCheckoutDir(), pluginCheckoutDir);
                }
            } else {
                // These hooks could redirect the SCM, skip checkout (if multiple plugins use the same preloaded repo)
                cloneFromSCM(pomData.getConnectionUrl(), plugin.name, plugin.version, pluginCheckoutDir);
            }
        } else {
            // If the plugin exists in a different directory (multimodule plugins)
            if (beforeCheckout.get("pluginDir") != null) {
                pluginCheckoutDir = (File) beforeCheckout.get("checkoutDir");
            }
            System.out.println("The plugin has already been checked out, likely due to a multimodule situation. Continue.");
        }
    } catch (ComponentLookupException e) {
        System.err.println("Error : " + e.getMessage());
        throw new PluginSourcesUnavailableException("Problem while creating ScmManager !", e);
    } catch (Exception e) {
        System.err.println("Error : " + e.getMessage());
        throw new PluginSourcesUnavailableException("Problem while checking out plugin sources!", e);
    }
    File buildLogFile = createBuildLogFile(config.reportFile, plugin.name, plugin.version, coreCoordinates);
    // Creating log directory
    FileUtils.forceMkdir(buildLogFile.getParentFile());
    // Creating log file
    FileUtils.fileWrite(buildLogFile.getAbsolutePath(), "");
    // Ran the BeforeCompileHooks
    Map<String, Object> beforeCompile = new HashMap<String, Object>();
    beforeCompile.put("pluginName", plugin.name);
    beforeCompile.put("plugin", plugin);
    beforeCompile.put("pluginDir", pluginCheckoutDir);
    beforeCompile.put("pomData", pomData);
    beforeCompile.put("config", config);
    beforeCompile.put("core", coreCoordinates);
    Map<String, Object> hookInfo = pcth.runBeforeCompilation(beforeCompile);
    boolean ranCompile = hookInfo.containsKey(PluginCompatTesterHookBeforeCompile.OVERRIDE_DEFAULT_COMPILE) ? (boolean) hookInfo.get(PluginCompatTesterHookBeforeCompile.OVERRIDE_DEFAULT_COMPILE) : false;
    try {
        // We also skip potential javadoc execution to avoid general test failure.
        if (!ranCompile) {
            runner.run(mconfig, pluginCheckoutDir, buildLogFile, "clean", "process-test-classes", "-Dmaven.javadoc.skip");
        }
        ranCompile = true;
        // Then transform the POM and run tests against that.
        // You might think that it would suffice to run e.g.
        // -Dmaven-surefire-plugin.version=2.15 -Dmaven.test.dependency.excludes=org.jenkins-ci.main:jenkins-war -Dmaven.test.additionalClasspath=/…/org/jenkins-ci/main/jenkins-war/1.580.1/jenkins-war-1.580.1.war clean test
        // (2.15+ required for ${maven.test.dependency.excludes} and ${maven.test.additionalClasspath} to be honored from CLI)
        // but it does not work; there are lots of linkage errors as some things are expected to be in the test classpath which are not.
        // Much simpler to do use the parent POM to set up the test classpath.
        MavenPom pom = new MavenPom(pluginCheckoutDir);
        try {
            addSplitPluginDependencies(plugin.name, mconfig, pluginCheckoutDir, pom, otherPlugins, pluginGroupIds, coreCoordinates.version);
        } catch (Exception x) {
            x.printStackTrace();
            pomData.getWarningMessages().add(Functions.printThrowable(x));
        // but continue
        }
        List<String> args = new ArrayList<String>();
        args.add("--define=maven.test.redirectTestOutputToFile=false");
        args.add("--define=concurrency=1");
        args.add("hpi:resolve-test-dependencies");
        args.add("hpi:test-hpl");
        args.add("surefire:test");
        // Run preexecution hooks
        Map<String, Object> forExecutionHooks = new HashMap<String, Object>();
        forExecutionHooks.put("pluginName", plugin.name);
        forExecutionHooks.put("args", args);
        forExecutionHooks.put("pomData", pomData);
        forExecutionHooks.put("pom", pom);
        forExecutionHooks.put("coreCoordinates", coreCoordinates);
        forExecutionHooks.put("config", config);
        forExecutionHooks.put("pluginDir", pluginCheckoutDir);
        pcth.runBeforeExecution(forExecutionHooks);
        runner.run(mconfig, pluginCheckoutDir, buildLogFile, ((List<String>) forExecutionHooks.get("args")).toArray(new String[args.size()]));
        return new TestExecutionResult(((PomData) forExecutionHooks.get("pomData")).getWarningMessages());
    } catch (PomExecutionException e) {
        PomExecutionException e2 = new PomExecutionException(e);
        e2.getPomWarningMessages().addAll(pomData.getWarningMessages());
        if (ranCompile) {
            // So the status is considered to be TEST_FAILURES not COMPILATION_ERROR:
            e2.succeededPluginArtifactIds.add("maven-compiler-plugin");
        }
        throw e2;
    }
}