Examples with ToolExecution ca.corefacility.bioinformatics.irida.model.workflow.analysis.ToolExecution used on opensource projects

Example 6 with ToolExecution

use of ca.corefacility.bioinformatics.irida.model.workflow.analysis.ToolExecution in project irida by phac-nml.

the class AnalysisServiceImplIT method testCreatePhylogenomicsAnalysis.

@Test
@WithMockUser(username = "admin", roles = "ADMIN")
public void testCreatePhylogenomicsAnalysis() throws IOException {
    Path treePath = Files.createTempFile(null, null);
    Path tablePath = Files.createTempFile(null, null);
    Path matrixPath = Files.createTempFile(null, null);
    Map<String, String> params = new HashMap<>();
    params.put("param", "value");
    ToolExecution toolExecutionTree = new ToolExecution(null, "ls", "1.0", "executionManagerId", params, "/bin/ls -lrth");
    ToolExecution toolExecutionTable = new ToolExecution(null, "ls", "1.0", "executionManagerId", params, "/bin/ls -lrth");
    ToolExecution toolExecutionMatrix = new ToolExecution(null, "ls", "1.0", "executionManagerId", params, "/bin/ls -lrth");
    AnalysisOutputFile tree = new AnalysisOutputFile(treePath, "internal-galaxy-tree-identifier", "", toolExecutionTree);
    AnalysisOutputFile table = new AnalysisOutputFile(tablePath, "internal-galaxy-table-identifier", "", toolExecutionTable);
    AnalysisOutputFile matrix = new AnalysisOutputFile(matrixPath, "internal-galaxy-matrix-identifier", "", toolExecutionMatrix);
    Map<String, AnalysisOutputFile> analysisOutputFiles = new ImmutableMap.Builder<String, AnalysisOutputFile>().put("tree", tree).put("matrix", matrix).put("table", table).build();
    Analysis pipeline = new Analysis(EXECUTION_MANAGER_ID, analysisOutputFiles, AnalysisType.PHYLOGENOMICS);
    // make sure that we're not falsely putting the files into the correct
    // directory in the first place.
    assertFalse("file was stored in the wrong directory.", pipeline.getAnalysisOutputFile(TREE_KEY).getFile().startsWith(outputFileBaseDirectory));
    assertFalse("file was stored in the wrong directory.", pipeline.getAnalysisOutputFile(MATRIX_KEY).getFile().startsWith(outputFileBaseDirectory));
    assertFalse("file was stored in the wrong directory.", pipeline.getAnalysisOutputFile(TABLE_KEY).getFile().startsWith(outputFileBaseDirectory));
    Analysis analysis = analysisService.create(pipeline);
    // make sure that we put the analysis output files into the correct
    // directory.
    assertEquals("returned analysis was of the wrong type.", AnalysisType.PHYLOGENOMICS, analysis.getAnalysisType());
    assertTrue("file was stored in the wrong directory.", analysis.getAnalysisOutputFile(TREE_KEY).getFile().startsWith(outputFileBaseDirectory));
    assertTrue("file was stored in the wrong directory.", analysis.getAnalysisOutputFile(MATRIX_KEY).getFile().startsWith(outputFileBaseDirectory));
    assertTrue("file was stored in the wrong directory.", analysis.getAnalysisOutputFile(TABLE_KEY).getFile().startsWith(outputFileBaseDirectory));
}

Example 7 with ToolExecution

use of ca.corefacility.bioinformatics.irida.model.workflow.analysis.ToolExecution in project irida by phac-nml.

the class TestDataFactory method constructAnalysisOutputFile.

private static AnalysisOutputFile constructAnalysisOutputFile(String name, Long id) {
    if (id == null) {
        id = 1L;
    }
    ToolExecution toolExecution = new ToolExecution(1L, null, "testTool", "0.0.12", "executionManagersId", ImmutableMap.of());
    final AnalysisOutputFile of = new AnalysisOutputFile(Paths.get(FAKE_FILE_PATH.replace("{name}", name)), "", FAKE_EXECUTION_MANAGER_ID, toolExecution);
    final DirectFieldAccessor dfa = new DirectFieldAccessor(of);
    dfa.setPropertyValue("id", id);
    return of;
}

Example 8 with ToolExecution

use of ca.corefacility.bioinformatics.irida.model.workflow.analysis.ToolExecution in project irida by phac-nml.

the class AnalysisProvenanceServiceGalaxy method buildToolExecutionForHistoryStep.

/**
 * Build up a complete *tree* of ToolExecution from Galaxy's history
 * contents provenance objects. Recursively follows predecessors from the
 * current history.
 *
 * @param toolDetails
 *            the details of the current tool to build up tool execution
 *            details for.
 * @param currentProvenance
 *            the provenance that corresponds to the tool details.
 * @param historyId
 *            the Galaxy ID we should use to extract tool execution
 *            information.
 * @return the entire tree of ToolExecutions for the tool and its
 *         provenance.
 * @throws ExecutionManagerException
 *             if we could not get the history contents provenance or the
 *             tool details for a predecessor of the current tool details or
 *             provenance.
 */
private ToolExecution buildToolExecutionForHistoryStep(final Tool toolDetails, final HistoryContentsProvenance currentProvenance, final String historyId) throws ExecutionManagerException {
    final Map<String, Set<String>> predecessors = getPredecessors(currentProvenance);
    final Map<String, Object> parameters = currentProvenance.getParameters();
    // remove keys from parameters that are Galaxy-related (and thus
    // ignorable), or keys that *match* input keys (as mentioned in
    // getPredecessors, the input keys are going to have a numeric
    // suffix and so don't equal the key that we want to remove from the
    // key set):
    /* @formatter:off */
    final Set<String> parameterKeys = parameters.keySet().stream().filter(k -> !PARAMETERS_TO_IGNORE.contains(k)).filter(k -> !predecessors.keySet().stream().anyMatch(p -> k.contains(p))).collect(Collectors.toSet());
    /* @formatter:on */
    final Map<String, Object> paramValues = new HashMap<>();
    for (final String parameterKey : parameterKeys) {
        paramValues.put(parameterKey, parameters.get(parameterKey));
    }
    final Set<ToolExecution> prevSteps = new HashSet<>();
    final String toolName = toolDetails.getName();
    final String toolVersion = toolDetails.getVersion();
    final String jobId = currentProvenance.getJobId();
    final JobDetails jobDetails = jobsClient.showJob(jobId);
    final String commandLine = jobDetails.getCommandLine();
    final Map<String, String> paramStrings = buildParamMap(paramValues);
    for (final String predecessorKey : predecessors.keySet()) {
        // arbitrarily select one of the predecessors from the set, then
        // recurse on that predecessor:
        final String predecessor = predecessors.get(predecessorKey).iterator().next();
        final HistoryContentsProvenance previousProvenance = galaxyHistoriesService.showProvenance(historyId, predecessor);
        final Tool previousToolDetails = toolsClient.showTool(previousProvenance.getToolId());
        final ToolExecution toolExecution = buildToolExecutionForHistoryStep(previousToolDetails, previousProvenance, historyId);
        prevSteps.add(toolExecution);
    }
    return new ToolExecution(prevSteps, toolName, toolVersion, jobId, paramStrings, commandLine);
}

Example 9 with ToolExecution

use of ca.corefacility.bioinformatics.irida.model.workflow.analysis.ToolExecution in project irida by phac-nml.

the class SNVPhylAnalysisIT method testSNVPhylSuccess.

/**
 * Tests out successfully executing the SNVPhyl pipeline.
 *
 * @throws Exception
 */
@Test
@WithMockUser(username = "aaron", roles = "ADMIN")
public void testSNVPhylSuccess() throws Exception {
    SequenceFilePair sequenceFilePairA = databaseSetupGalaxyITService.setupSampleSequenceFileInDatabase(1L, sequenceFilePathsA1List, sequenceFilePathsA2List).get(0);
    SequenceFilePair sequenceFilePairB = databaseSetupGalaxyITService.setupSampleSequenceFileInDatabase(2L, sequenceFilePathsB1List, sequenceFilePathsB2List).get(0);
    SequenceFilePair sequenceFilePairC = databaseSetupGalaxyITService.setupSampleSequenceFileInDatabase(3L, sequenceFilePathsC1List, sequenceFilePathsC2List).get(0);
    Map<String, String> parameters = ImmutableMap.of("snv-abundance-ratio", "0.75", "minimum-read-coverage", "2", "filter-density-threshold", "2", "filter-density-window-size", "3");
    waitForFilesToSettle(sequenceFilePairA, sequenceFilePairB, sequenceFilePairC);
    AnalysisSubmission submission = databaseSetupGalaxyITService.setupPairSubmissionInDatabase(Sets.newHashSet(sequenceFilePairA, sequenceFilePairB, sequenceFilePairC), referenceFilePath, parameters, snvPhylWorkflow.getWorkflowIdentifier());
    completeSubmittedAnalyses(submission.getId());
    submission = analysisSubmissionRepository.findOne(submission.getId());
    assertEquals("analysis state should be completed.", AnalysisState.COMPLETED, submission.getAnalysisState());
    Analysis analysisPhylogenomics = submission.getAnalysis();
    assertEquals("Should have generated a phylogenomics pipeline analysis type.", AnalysisType.PHYLOGENOMICS, analysisPhylogenomics.getAnalysisType());
    assertEquals("the phylogenomics pipeline should have 8 output files.", 8, analysisPhylogenomics.getAnalysisOutputFiles().size());
    @SuppressWarnings("resource") String matrixContent = new Scanner(analysisPhylogenomics.getAnalysisOutputFile(MATRIX_KEY).getFile().toFile()).useDelimiter("\\Z").next();
    assertTrue("snpMatrix should be the same but is \"" + matrixContent + "\"", com.google.common.io.Files.equal(outputSnvMatrix1.toFile(), analysisPhylogenomics.getAnalysisOutputFile(MATRIX_KEY).getFile().toFile()));
    assertNotNull("file should have tool provenance attached.", analysisPhylogenomics.getAnalysisOutputFile(MATRIX_KEY).getCreatedByTool());
    @SuppressWarnings("resource") String snpTableContent = new Scanner(analysisPhylogenomics.getAnalysisOutputFile(TABLE_KEY).getFile().toFile()).useDelimiter("\\Z").next();
    assertTrue("snpTable should be the same but is \"" + snpTableContent + "\"", com.google.common.io.Files.equal(outputSnvTable1.toFile(), analysisPhylogenomics.getAnalysisOutputFile(TABLE_KEY).getFile().toFile()));
    assertNotNull("file should have tool provenance attached.", analysisPhylogenomics.getAnalysisOutputFile(TABLE_KEY).getCreatedByTool());
    @SuppressWarnings("resource") String vcf2coreContent = new Scanner(analysisPhylogenomics.getAnalysisOutputFile(CORE_KEY).getFile().toFile()).useDelimiter("\\Z").next();
    assertTrue("vcf2core should be the same but is \"" + vcf2coreContent + "\"", com.google.common.io.Files.equal(vcf2core1.toFile(), analysisPhylogenomics.getAnalysisOutputFile(CORE_KEY).getFile().toFile()));
    assertNotNull("file should have tool provenance attached.", analysisPhylogenomics.getAnalysisOutputFile(CORE_KEY).getCreatedByTool());
    // only check size of mapping quality file due to samples output in random order
    assertTrue("the mapping quality file should not be empty.", Files.size(analysisPhylogenomics.getAnalysisOutputFile(QUALITY_KEY).getFile()) > 0);
    @SuppressWarnings("resource") String filterStatsContent = new Scanner(analysisPhylogenomics.getAnalysisOutputFile(STATS_KEY).getFile().toFile()).useDelimiter("\\Z").next();
    assertTrue("filterStats should be the same but is \"" + filterStatsContent + "\"", com.google.common.io.Files.equal(filterStats1.toFile(), analysisPhylogenomics.getAnalysisOutputFile(STATS_KEY).getFile().toFile()));
    assertNotNull("file should have tool provenance attached.", analysisPhylogenomics.getAnalysisOutputFile(STATS_KEY).getCreatedByTool());
    @SuppressWarnings("resource") String snvAlignContent = new Scanner(analysisPhylogenomics.getAnalysisOutputFile(ALIGN_KEY).getFile().toFile()).useDelimiter("\\Z").next();
    assertTrue("snvAlign should be the same but is \"" + snvAlignContent + "\"", com.google.common.io.Files.equal(snvAlign1.toFile(), analysisPhylogenomics.getAnalysisOutputFile(ALIGN_KEY).getFile().toFile()));
    assertNotNull("file should have tool provenance attached.", analysisPhylogenomics.getAnalysisOutputFile(ALIGN_KEY).getCreatedByTool());
    // only test to make sure the files have a valid size since PhyML uses a
    // random seed to generate the tree (and so changes results)
    assertTrue("the phylogenetic tree file should not be empty.", Files.size(analysisPhylogenomics.getAnalysisOutputFile(TREE_KEY).getFile()) > 0);
    assertTrue("the phylogenetic tree stats file should not be empty.", Files.size(analysisPhylogenomics.getAnalysisOutputFile(TREE_KEY).getFile()) > 0);
    // try to follow the phylogenomics provenance all the way back to the
    // upload tools
    final List<ToolExecution> toolsToVisit = Lists.newArrayList(analysisPhylogenomics.getAnalysisOutputFile(TREE_KEY).getCreatedByTool());
    assertFalse("file should have tool provenance attached.", toolsToVisit.isEmpty());
    boolean foundReadsInputTool = false;
    boolean foundReferenceInputTool = false;
    // one where you upload the reads.
    while (!toolsToVisit.isEmpty()) {
        final ToolExecution ex = toolsToVisit.remove(0);
        toolsToVisit.addAll(ex.getPreviousSteps());
        if (ex.isInputTool()) {
            final Map<String, String> params = ex.getExecutionTimeParameters();
            logger.debug("Input tool has " + params);
            foundReferenceInputTool |= params.containsKey("files.NAME") && params.get("files.NAME").contains("reference") && params.get("file_type").contains("fasta");
            foundReadsInputTool |= params.get("file_type").contains("fastq");
        }
    }
    assertTrue("Should have found both reads and reference input tools.", foundReadsInputTool && foundReferenceInputTool);
}

Example 10 with ToolExecution

use of ca.corefacility.bioinformatics.irida.model.workflow.analysis.ToolExecution in project irida by phac-nml.

the class SNVPhylAnalysisIT method testSNVPhylSuccessRemoveSNVDensity.

/**
 * Tests out successfully executing the SNVPhyl pipeline and passing a lower value for SNV density threshold to filter out SNVs.
 *
 * @throws Exception
 */
@Test
@WithMockUser(username = "aaron", roles = "ADMIN")
public void testSNVPhylSuccessRemoveSNVDensity() throws Exception {
    SequenceFilePair sequenceFilePairA = databaseSetupGalaxyITService.setupSampleSequenceFileInDatabase(1L, sequenceFilePathsA1List, sequenceFilePathsA2List).get(0);
    SequenceFilePair sequenceFilePairB = databaseSetupGalaxyITService.setupSampleSequenceFileInDatabase(2L, sequenceFilePathsB1List, sequenceFilePathsB2List).get(0);
    SequenceFilePair sequenceFilePairC = databaseSetupGalaxyITService.setupSampleSequenceFileInDatabase(3L, sequenceFilePathsC1List, sequenceFilePathsC2List).get(0);
    Map<String, String> parameters = ImmutableMap.of("snv-abundance-ratio", "0.75", "minimum-read-coverage", "2", "filter-density-threshold", "2", "filter-density-window-size", "4");
    AnalysisSubmission submission = databaseSetupGalaxyITService.setupPairSubmissionInDatabase(Sets.newHashSet(sequenceFilePairA, sequenceFilePairB, sequenceFilePairC), referenceFilePath, parameters, snvPhylWorkflow.getWorkflowIdentifier());
    completeSubmittedAnalyses(submission.getId());
    submission = analysisSubmissionRepository.findOne(submission.getId());
    assertEquals("analysis state should be completed.", AnalysisState.COMPLETED, submission.getAnalysisState());
    Analysis analysisPhylogenomics = submission.getAnalysis();
    assertEquals("Should have generated a phylogenomics pipeline analysis type.", AnalysisType.PHYLOGENOMICS, analysisPhylogenomics.getAnalysisType());
    assertEquals("the phylogenomics pipeline should have 8 output files.", 8, analysisPhylogenomics.getAnalysisOutputFiles().size());
    @SuppressWarnings("resource") String matrixContent = new Scanner(analysisPhylogenomics.getAnalysisOutputFile(MATRIX_KEY).getFile().toFile()).useDelimiter("\\Z").next();
    assertTrue("snpMatrix should be the same but is \"" + matrixContent + "\"", com.google.common.io.Files.equal(outputSnvMatrix3.toFile(), analysisPhylogenomics.getAnalysisOutputFile(MATRIX_KEY).getFile().toFile()));
    assertNotNull("file should have tool provenance attached.", analysisPhylogenomics.getAnalysisOutputFile(MATRIX_KEY).getCreatedByTool());
    @SuppressWarnings("resource") String snpTableContent = new Scanner(analysisPhylogenomics.getAnalysisOutputFile(TABLE_KEY).getFile().toFile()).useDelimiter("\\Z").next();
    assertTrue("snpTable should be the same but is \"" + snpTableContent + "\"", com.google.common.io.Files.equal(outputSnvTable3.toFile(), analysisPhylogenomics.getAnalysisOutputFile(TABLE_KEY).getFile().toFile()));
    assertNotNull("file should have tool provenance attached.", analysisPhylogenomics.getAnalysisOutputFile(TABLE_KEY).getCreatedByTool());
    @SuppressWarnings("resource") String vcf2coreContent = new Scanner(analysisPhylogenomics.getAnalysisOutputFile(CORE_KEY).getFile().toFile()).useDelimiter("\\Z").next();
    assertTrue("vcf2core should be the same but is \"" + vcf2coreContent + "\"", com.google.common.io.Files.equal(vcf2core3.toFile(), analysisPhylogenomics.getAnalysisOutputFile(CORE_KEY).getFile().toFile()));
    assertNotNull("file should have tool provenance attached.", analysisPhylogenomics.getAnalysisOutputFile(CORE_KEY).getCreatedByTool());
    // only check size of mapping quality file due to samples output in random order
    assertTrue("the mapping quality file should not be empty.", Files.size(analysisPhylogenomics.getAnalysisOutputFile(QUALITY_KEY).getFile()) > 0);
    @SuppressWarnings("resource") String filterStatsContent = new Scanner(analysisPhylogenomics.getAnalysisOutputFile(STATS_KEY).getFile().toFile()).useDelimiter("\\Z").next();
    assertTrue("filterStats should be the same but is \"" + filterStatsContent + "\"", com.google.common.io.Files.equal(filterStats3.toFile(), analysisPhylogenomics.getAnalysisOutputFile(STATS_KEY).getFile().toFile()));
    assertNotNull("file should have tool provenance attached.", analysisPhylogenomics.getAnalysisOutputFile(STATS_KEY).getCreatedByTool());
    @SuppressWarnings("resource") String snvAlignContent = new Scanner(analysisPhylogenomics.getAnalysisOutputFile(ALIGN_KEY).getFile().toFile()).useDelimiter("\\Z").next();
    assertTrue("snvAlign should be the same but is \"" + snvAlignContent + "\"", com.google.common.io.Files.equal(snvAlign3.toFile(), analysisPhylogenomics.getAnalysisOutputFile(ALIGN_KEY).getFile().toFile()));
    assertNotNull("file should have tool provenance attached.", analysisPhylogenomics.getAnalysisOutputFile(ALIGN_KEY).getCreatedByTool());
    // only test to make sure the files have a valid size since PhyML uses a
    // random seed to generate the tree (and so changes results)
    assertTrue("the phylogenetic tree file should not be empty.", Files.size(analysisPhylogenomics.getAnalysisOutputFile(TREE_KEY).getFile()) > 0);
    assertTrue("the phylogenetic tree stats file should not be empty.", Files.size(analysisPhylogenomics.getAnalysisOutputFile(TREE_KEY).getFile()) > 0);
    // try to follow the phylogenomics provenance all the way back to the
    // upload tools
    List<ToolExecution> toolsToVisit = Lists.newArrayList(analysisPhylogenomics.getAnalysisOutputFile(TREE_KEY).getCreatedByTool());
    assertFalse("file should have tool provenance attached.", toolsToVisit.isEmpty());
    String minVcf2AlignCov = null;
    String altAlleleFraction = null;
    String minimumPercentCoverage = null;
    String minimumDepthVerify = null;
    String filterDensityThreshold = null;
    String filterDensityWindowSize = null;
    // one where you upload the reads.
    while (!toolsToVisit.isEmpty()) {
        final ToolExecution ex = toolsToVisit.remove(0);
        toolsToVisit.addAll(ex.getPreviousSteps());
        if (ex.getToolName().contains("Consolidate VCFs")) {
            final Map<String, String> params = ex.getExecutionTimeParameters();
            minVcf2AlignCov = params.get("coverage");
            altAlleleFraction = params.get("snv_abundance_ratio");
            filterDensityThreshold = params.get("use_density_filter.threshold");
            filterDensityWindowSize = params.get("use_density_filter.window_size");
            break;
        }
    }
    // try to follow the mapping quality provenance all the way back to the
    // upload tools
    toolsToVisit = Lists.newArrayList(analysisPhylogenomics.getAnalysisOutputFile(QUALITY_KEY).getCreatedByTool());
    assertFalse("file should have tool provenance attached.", toolsToVisit.isEmpty());
    while (!toolsToVisit.isEmpty()) {
        final ToolExecution ex = toolsToVisit.remove(0);
        toolsToVisit.addAll(ex.getPreviousSteps());
        if (ex.getToolName().contains("Verify Mapping Quality")) {
            final Map<String, String> params = ex.getExecutionTimeParameters();
            minimumPercentCoverage = params.get("minmap");
            minimumDepthVerify = params.get("mindepth");
        }
    }
    assertEquals("incorrect minimum vcf 2 align coverage", "\"2\"", minVcf2AlignCov);
    assertEquals("incorrect alternative allele fraction", "\"0.75\"", altAlleleFraction);
    assertEquals("incorrect minimum depth for verify map", "\"2\"", minimumDepthVerify);
    assertEquals("incorrect min percent coverage for verify map", "\"80\"", minimumPercentCoverage);
    assertEquals("incorrect filter density threshold", "2", filterDensityThreshold);
    assertEquals("incorrect filter density window size", "4", filterDensityWindowSize);
}