Examples with ProcessController org.broadinstitute.hellbender.utils.runtime.ProcessController used on opensource projects

Example 1 with ProcessController

use of org.broadinstitute.hellbender.utils.runtime.ProcessController in project gatk by broadinstitute.

the class GenotypeGVCFsIntegrationTest method compareToGATK3.

/*
    This test is useful for testing changes in GATK4 versus different versions of GATK3.
    To use, set GATK3_PATH to point to a particular version of gatk, then enable this test and run.

    It will cache the gatk3 outputs in a folder called gatk3results, it does it's best to avoid reusing bad results by
    comparing the md5 of the gatk3 path, input file path, reference, and commandline, but it doesn't know about internal changes to files
    You have to manually delete the cache if you make changes inside the input files.

    The expected outputs based on gatk3's results will be put in a folder called expectedResults.
    These are overwritten during each test, the names are based on the name of the existing expected output file.

    This method should be removed after GenotypeGVCFs has been completely validated against GATK3.
     */
@Test(dataProvider = "gvcfsToGenotype", enabled = false)
public void compareToGATK3(File input, File outputFile, List<String> extraArgs, String reference) throws IOException, NoSuchAlgorithmException {
    final String GATK3_PATH = "gatk3.7-30-ga4f720357.jar";
    final String params = GATK3_PATH + input.getAbsolutePath() + extraArgs.stream().collect(Collectors.joining()) + reference;
    final String md5 = DigestUtils.md5Hex(params);
    final File gatk3ResultsDir = new File("gatk3results");
    if (!gatk3ResultsDir.exists()) {
        Assert.assertTrue(gatk3ResultsDir.mkdir());
    }
    final File gatk3Result = new File(gatk3ResultsDir, md5 + ".vcf");
    if (!gatk3Result.exists()) {
        List<String> gatk3Command = new ArrayList<>(Arrays.asList("java", "-jar", GATK3_PATH, "-T", "GenotypeGVCFs"));
        gatk3Command.add("-V");
        gatk3Command.add(input.getAbsolutePath());
        gatk3Command.add("-o");
        gatk3Command.add(gatk3Result.getAbsolutePath());
        gatk3Command.add("-R");
        gatk3Command.add(reference);
        gatk3Command.addAll(extraArgs);
        runProcess(new ProcessController(), gatk3Command.toArray(new String[gatk3Command.size()]));
    } else {
        System.out.println("Found precomputed gatk3Result");
    }
    final Path expectedResultsDir = Paths.get("expectedResults");
    if (!Files.exists(expectedResultsDir)) {
        Files.createDirectory(expectedResultsDir);
    }
    Files.copy(gatk3Result.toPath(), expectedResultsDir.resolve(outputFile.getName()), StandardCopyOption.REPLACE_EXISTING);
    assertGenotypesMatch(input, gatk3Result, extraArgs, reference);
    assertVariantContextsMatch(input, gatk3Result, extraArgs, reference);
}

Example 2 with ProcessController

use of org.broadinstitute.hellbender.utils.runtime.ProcessController in project gatk by broadinstitute.

the class RScriptExecutor method exec.

public boolean exec() {
    if (!RSCRIPT_EXISTS) {
        if (!ignoreExceptions) {
            throw new UserException.CannotExecuteRScript(RSCRIPT_MISSING_MESSAGE);
        } else {
            logger.warn("Skipping: " + getApproximateCommandLine());
            return false;
        }
    }
    List<File> tempFiles = new ArrayList<>();
    try {
        File tempLibSourceDir = IOUtils.tempDir("RlibSources.", "");
        File tempLibInstallationDir = IOUtils.tempDir("Rlib.", "");
        tempFiles.add(tempLibSourceDir);
        tempFiles.add(tempLibInstallationDir);
        StringBuilder expression = new StringBuilder("tempLibDir = '").append(tempLibInstallationDir).append("';");
        if (!this.libraries.isEmpty()) {
            List<String> tempLibraryPaths = new ArrayList<>();
            for (RScriptLibrary library : this.libraries) {
                File tempLibrary = library.writeLibrary(tempLibSourceDir);
                tempFiles.add(tempLibrary);
                tempLibraryPaths.add(tempLibrary.getAbsolutePath());
            }
            expression.append("install.packages(");
            expression.append("pkgs=c('").append(StringUtils.join(tempLibraryPaths, "', '")).append("'), lib=tempLibDir, repos=NULL, type='source', ");
            // Install faster by eliminating cruft.
            expression.append("INSTALL_opts=c('--no-libs', '--no-data', '--no-help', '--no-demo', '--no-exec')");
            expression.append(");");
            for (RScriptLibrary library : this.libraries) {
                expression.append("library('").append(library.getLibraryName()).append("', lib.loc=tempLibDir);");
            }
        }
        for (Resource script : this.scriptResources) {
            File tempScript = IOUtils.writeTempResource(script);
            tempFiles.add(tempScript);
            expression.append("source('").append(tempScript.getAbsolutePath()).append("');");
        }
        for (File script : this.scriptFiles) {
            expression.append("source('").append(script.getAbsolutePath()).append("');");
        }
        String[] cmd = new String[this.args.size() + 3];
        int i = 0;
        cmd[i++] = RSCRIPT_BINARY;
        cmd[i++] = "-e";
        cmd[i++] = expression.toString();
        for (String arg : this.args) cmd[i++] = arg;
        ProcessSettings processSettings = new ProcessSettings(cmd);
        //if debug is enabled, output the stdout and stdder, otherwise capture it to a buffer
        if (logger.isDebugEnabled()) {
            processSettings.getStdoutSettings().printStandard(true);
            processSettings.getStderrSettings().printStandard(true);
        } else {
            processSettings.getStdoutSettings().setBufferSize(8192);
            processSettings.getStderrSettings().setBufferSize(8192);
        }
        ProcessController controller = ProcessController.getThreadLocal();
        if (logger.isDebugEnabled()) {
            logger.debug("Executing:");
            for (String arg : cmd) logger.debug("  " + arg);
        }
        ProcessOutput po = controller.exec(processSettings);
        int exitValue = po.getExitValue();
        logger.debug("Result: " + exitValue);
        if (exitValue != 0) {
            StringBuilder message = new StringBuilder();
            message.append(String.format("\nRscript exited with %d\nCommand Line: %s", exitValue, String.join(" ", cmd)));
            //if debug was enabled the stdout/error were already output somewhere
            if (!logger.isDebugEnabled()) {
                message.append(String.format("\nStdout: %s\nStderr: %s", po.getStdout().getBufferString(), po.getStderr().getBufferString()));
            }
            throw new RScriptExecutorException(message.toString());
        }
        return true;
    } catch (GATKException e) {
        if (!ignoreExceptions) {
            throw e;
        } else {
            logger.warn(e.getMessage());
            return false;
        }
    } finally {
        for (File temp : tempFiles) FileUtils.deleteQuietly(temp);
    }
}

Example 3 with ProcessController

use of org.broadinstitute.hellbender.utils.runtime.ProcessController in project gatk-protected by broadinstitute.

the class GenotypeGVCFsIntegrationTest method compareToGATK3.

/*
    This test is useful for testing changes in GATK4 versus different versions of GATK3.
    To use, set GATK3_PATH to point to a particular version of gatk, then enable this test and run.

    It will cache the gatk3 outputs in a folder called gatk3results, it does it's best to avoid reusing bad results by
    comparing the md5 of the gatk3 path, input file path, reference, and commandline, but it doesn't know about internal changes to files
    You have to manually delete the cache if you make changes inside the input files.

    The expected outputs based on gatk3's results will be put in a folder called expectedResults.
    These are overwritten during each test, the names are based on the name of the existing expected output file.

    This method should be removed after GenotypeGVCFs has been completely validated against GATK3.
     */
@Test(dataProvider = "gvcfsToGenotype", enabled = false)
public void compareToGATK3(File input, File outputFile, List<String> extraArgs, String reference) throws IOException, NoSuchAlgorithmException {
    final String GATK3_PATH = "gatk3.7-30-ga4f720357.jar";
    final String params = GATK3_PATH + input.getAbsolutePath() + extraArgs.stream().collect(Collectors.joining()) + reference;
    final String md5 = DigestUtils.md5Hex(params);
    final File gatk3ResultsDir = new File("gatk3results");
    if (!gatk3ResultsDir.exists()) {
        Assert.assertTrue(gatk3ResultsDir.mkdir());
    }
    final File gatk3Result = new File(gatk3ResultsDir, md5 + ".vcf");
    if (!gatk3Result.exists()) {
        List<String> gatk3Command = new ArrayList<>(Arrays.asList("java", "-jar", GATK3_PATH, "-T", "GenotypeGVCFs"));
        gatk3Command.add("-V");
        gatk3Command.add(input.getAbsolutePath());
        gatk3Command.add("-o");
        gatk3Command.add(gatk3Result.getAbsolutePath());
        gatk3Command.add("-R");
        gatk3Command.add(reference);
        gatk3Command.addAll(extraArgs);
        runProcess(new ProcessController(), gatk3Command.toArray(new String[gatk3Command.size()]));
    } else {
        System.out.println("Found precomputed gatk3Result");
    }
    final Path expectedResultsDir = Paths.get("expectedResults");
    if (!Files.exists(expectedResultsDir)) {
        Files.createDirectory(expectedResultsDir);
    }
    Files.copy(gatk3Result.toPath(), expectedResultsDir.resolve(outputFile.getName()), StandardCopyOption.REPLACE_EXISTING);
    assertGenotypesMatch(input, gatk3Result, extraArgs, reference);
    assertVariantContextsMatch(input, gatk3Result, extraArgs, reference);
}