Search in sources :

Example 26 with MetricsFile

use of htsjdk.samtools.metrics.MetricsFile in project gatk by broadinstitute.

the class PreAdapterOrientationScorerUnitTest method testBasicCounting.

/**
     * Note that (due to raw data), this test includes collapsing over libraries (not just the contexts).
     * @throws IOException
     */
@Test
public void testBasicCounting() throws IOException {
    final MetricsFile<SequencingArtifactMetrics.PreAdapterDetailMetrics, Comparable<?>> mf = new MetricsFile<>();
    mf.read(new FileReader(testPreAdapterDetailsMetrics));
    final Map<Transition, RealMatrix> countMap = PreAdapterOrientationScorer.countOrientationBiasMetricsOverContext(mf.getMetrics());
    Assert.assertNotNull(countMap);
    Assert.assertEquals(countMap.keySet().size(), 12);
    // Ground truth values painstakingly derived manually from both libraries (hence four values in the expression instead of two)
    // L1 norm can be used since all values are always positive.
    Assert.assertEquals(countMap.get(Transition.transitionOf('A', 'C')).getRowVector(0).getL1Norm(), 2836660 + 2631203 + 240 + 246, 1e-1);
    Assert.assertEquals(countMap.get(Transition.transitionOf('A', 'C')).getRowVector(1).getL1Norm(), 2852491 + 333 + 2646654 + 297, 1e-1);
    Assert.assertEquals(countMap.get(Transition.transitionOf('A', 'G')).getRowVector(0).getL1Norm(), 2631203 + 416 + 2836660 + 481, 1e-1);
    Assert.assertEquals(countMap.get(Transition.transitionOf('A', 'G')).getRowVector(1).getL1Norm(), 2852491 + 404 + 2646654 + 397, 1e-1);
}
Also used : MetricsFile(htsjdk.samtools.metrics.MetricsFile) RealMatrix(org.apache.commons.math3.linear.RealMatrix) Transition(org.broadinstitute.hellbender.tools.picard.analysis.artifacts.Transition) FileReader(java.io.FileReader) BaseTest(org.broadinstitute.hellbender.utils.test.BaseTest) Test(org.testng.annotations.Test)

Example 27 with MetricsFile

use of htsjdk.samtools.metrics.MetricsFile in project gatk by broadinstitute.

the class CollectQualityYieldMetrics method doWork.

/**
     * Main method for the program.  Checks that all input files are present and
     * readable and that the output file can be written to.  Then iterates through
     * all the records accumulating metrics.  Finally writes metrics file
     */
@Override
protected Object doWork() {
    final ProgressLogger progress = new ProgressLogger(logger);
    File output = new File(qualityYieldMetricsArgs.output);
    // Some quick parameter checking
    IOUtil.assertFileIsReadable(INPUT);
    IOUtil.assertFileIsWritable(output);
    logger.info("Reading input file and calculating metrics.");
    final SamReader sam = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT);
    final MetricsFile<QualityYieldMetrics, Integer> metricsFile = getMetricsFile();
    final QualityYieldMetrics metrics = new QualityYieldMetrics();
    metrics.setUseOriginalQualities(qualityYieldMetricsArgs.useOriginalQualities);
    for (final SAMRecord rec : sam) {
        metrics.addRead(new SAMRecordToGATKReadAdapter(rec));
        progress.record(rec);
    }
    metrics.finish();
    metricsFile.addMetric(metrics);
    metricsFile.write(output);
    return null;
}
Also used : SamReader(htsjdk.samtools.SamReader) QualityYieldMetrics(org.broadinstitute.hellbender.metrics.QualityYieldMetrics) SAMRecordToGATKReadAdapter(org.broadinstitute.hellbender.utils.read.SAMRecordToGATKReadAdapter) SAMRecord(htsjdk.samtools.SAMRecord) ProgressLogger(org.broadinstitute.hellbender.utils.runtime.ProgressLogger) MetricsFile(htsjdk.samtools.metrics.MetricsFile) File(java.io.File)

Example 28 with MetricsFile

use of htsjdk.samtools.metrics.MetricsFile in project gatk by broadinstitute.

the class GenotypeConcordanceTest method assertMetricsFileEqual.

private void assertMetricsFileEqual(final File actualMetricsFile, final File expectedMetricsFile) throws FileNotFoundException {
    // Actual metrics file
    final MetricsFile<GenotypeConcordanceSummaryMetrics, Comparable<?>> actual = new MetricsFile<>();
    actual.read(new FileReader(actualMetricsFile));
    // Expected metrics file
    final MetricsFile<GenotypeConcordanceSummaryMetrics, Comparable<?>> expected = new MetricsFile<>();
    expected.read(new FileReader(expectedMetricsFile));
    // Note - cannot use .equals as it calls .areHeadersEqual and they are not since the timestamp (at a minimum is different)
    Assert.assertTrue(expected.areMetricsEqual(actual));
    Assert.assertTrue(expected.areHistogramsEqual(actual));
}
Also used : MetricsFile(htsjdk.samtools.metrics.MetricsFile) FileReader(java.io.FileReader)

Example 29 with MetricsFile

use of htsjdk.samtools.metrics.MetricsFile in project gatk-protected by broadinstitute.

the class FilterByOrientationBias method onTraversalStart.

@Override
public void onTraversalStart() {
    // Gets around issue 2274 in gatk public
    if (transitions.size() == 0) {
        transitions.add(DEFAULT_ARTIFACT_MODE);
    }
    // Sort the input artifacts argument
    transitions.sort(null);
    final MetricsFile<SequencingArtifactMetrics.PreAdapterDetailMetrics, Comparable<?>> mf = new MetricsFile<>();
    try {
        mf.read(new FileReader(preAdapterMetricsFile));
    } catch (final FileNotFoundException fnfe) {
        throw new UserException("Could not find file: " + preAdapterMetricsFile.getAbsolutePath());
    }
    // Collect all of the transitions that were specified in the parameters.
    relevantTransitions.addAll(transitions.stream().map(s -> convertParameterToTransition(s)).collect(Collectors.toSet()));
    // Get the PreAdapterQ score from the picard metrics tool, which gives an indication of how badly infested the bam file is.
    transitionToPreAdapterScoreMap = PreAdapterOrientationScorer.scoreOrientationBiasMetricsOverContext(mf.getMetrics());
    logger.info("preAdapter scores:");
    transitionToPreAdapterScoreMap.keySet().stream().forEach(k -> logger.info(k + ": " + transitionToPreAdapterScoreMap.get(k)));
    setupVCFWriter();
}
Also used : MetricsFile(htsjdk.samtools.metrics.MetricsFile) FileNotFoundException(java.io.FileNotFoundException) FileReader(java.io.FileReader) UserException(org.broadinstitute.hellbender.exceptions.UserException)

Example 30 with MetricsFile

use of htsjdk.samtools.metrics.MetricsFile in project gatk by broadinstitute.

the class MarkDuplicatesSparkIntegrationTest method testMarkDuplicatesSparkIntegrationTestLocal.

@Test(groups = "spark", dataProvider = "md")
public void testMarkDuplicatesSparkIntegrationTestLocal(final File input, final long totalExpected, final long dupsExpected, Map<String, List<String>> metricsExpected) throws IOException {
    ArgumentsBuilder args = new ArgumentsBuilder();
    args.add("--" + StandardArgumentDefinitions.INPUT_LONG_NAME);
    args.add(input.getPath());
    args.add("--" + StandardArgumentDefinitions.OUTPUT_LONG_NAME);
    File outputFile = createTempFile("markdups", ".bam");
    outputFile.delete();
    args.add(outputFile.getAbsolutePath());
    args.add("--METRICS_FILE");
    File metricsFile = createTempFile("markdups_metrics", ".txt");
    args.add(metricsFile.getAbsolutePath());
    runCommandLine(args.getArgsArray());
    Assert.assertTrue(outputFile.exists(), "Can't find expected MarkDuplicates output file at " + outputFile.getAbsolutePath());
    int totalReads = 0;
    int duplicateReads = 0;
    try (final ReadsDataSource outputReads = new ReadsDataSource(outputFile.toPath())) {
        for (GATKRead read : outputReads) {
            ++totalReads;
            if (read.isDuplicate()) {
                ++duplicateReads;
            }
        }
    }
    Assert.assertEquals(totalReads, totalExpected, "Wrong number of reads in output BAM");
    Assert.assertEquals(duplicateReads, dupsExpected, "Wrong number of duplicate reads in output BAM");
    final MetricsFile<DuplicationMetrics, Comparable<?>> metricsOutput = new MetricsFile<>();
    try {
        metricsOutput.read(new FileReader(metricsFile));
    } catch (final FileNotFoundException ex) {
        System.err.println("Metrics file not found: " + ex);
    }
    final List<DuplicationMetrics> nonEmptyMetrics = metricsOutput.getMetrics().stream().filter(metric -> metric.UNPAIRED_READS_EXAMINED != 0L || metric.READ_PAIRS_EXAMINED != 0L || metric.UNMAPPED_READS != 0L || metric.UNPAIRED_READ_DUPLICATES != 0L || metric.READ_PAIR_DUPLICATES != 0L || metric.READ_PAIR_OPTICAL_DUPLICATES != 0L || (metric.PERCENT_DUPLICATION != null && metric.PERCENT_DUPLICATION != 0.0 && !Double.isNaN(metric.PERCENT_DUPLICATION)) || (metric.ESTIMATED_LIBRARY_SIZE != null && metric.ESTIMATED_LIBRARY_SIZE != 0L)).collect(Collectors.toList());
    Assert.assertEquals(nonEmptyMetrics.size(), metricsExpected.size(), "Wrong number of metrics with non-zero fields.");
    for (int i = 0; i < nonEmptyMetrics.size(); i++) {
        final DuplicationMetrics observedMetrics = nonEmptyMetrics.get(i);
        List<?> expectedList = metricsExpected.get(observedMetrics.LIBRARY);
        Assert.assertNotNull(expectedList, "Unexpected library found: " + observedMetrics.LIBRARY);
        Assert.assertEquals(observedMetrics.UNPAIRED_READS_EXAMINED, expectedList.get(0));
        Assert.assertEquals(observedMetrics.READ_PAIRS_EXAMINED, expectedList.get(1));
        Assert.assertEquals(observedMetrics.UNMAPPED_READS, expectedList.get(2));
        Assert.assertEquals(observedMetrics.UNPAIRED_READ_DUPLICATES, expectedList.get(3));
        Assert.assertEquals(observedMetrics.READ_PAIR_DUPLICATES, expectedList.get(4));
        Assert.assertEquals(observedMetrics.READ_PAIR_OPTICAL_DUPLICATES, expectedList.get(5));
        Assert.assertEquals(observedMetrics.PERCENT_DUPLICATION, expectedList.get(6));
        //so we work around it by passing in an 'expected 0L' and only comparing to it if the actual value is non-null
        if (observedMetrics.ESTIMATED_LIBRARY_SIZE != null && (Long) expectedList.get(7) != 0L) {
            Assert.assertEquals(observedMetrics.ESTIMATED_LIBRARY_SIZE, expectedList.get(7));
        }
    }
}
Also used : GATKRead(org.broadinstitute.hellbender.utils.read.GATKRead) MetricsFile(htsjdk.samtools.metrics.MetricsFile) ReadsDataSource(org.broadinstitute.hellbender.engine.ReadsDataSource) MarkDuplicatesSparkTester(org.broadinstitute.hellbender.utils.read.markduplicates.MarkDuplicatesSparkTester) MarkDuplicatesIntegrationTest(org.broadinstitute.hellbender.tools.picard.sam.markduplicates.MarkDuplicatesIntegrationTest) CommandLineProgram(org.broadinstitute.hellbender.cmdline.CommandLineProgram) DuplicationMetrics(org.broadinstitute.hellbender.utils.read.markduplicates.DuplicationMetrics) DataProvider(org.testng.annotations.DataProvider) ImmutableMap(com.google.common.collect.ImmutableMap) StandardArgumentDefinitions(org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions) ArgumentsBuilder(org.broadinstitute.hellbender.utils.test.ArgumentsBuilder) Test(org.testng.annotations.Test) IOException(java.io.IOException) MetricsFile(htsjdk.samtools.metrics.MetricsFile) GATKRead(org.broadinstitute.hellbender.utils.read.GATKRead) Collectors(java.util.stream.Collectors) File(java.io.File) FileNotFoundException(java.io.FileNotFoundException) MarkDuplicatesSpark(org.broadinstitute.hellbender.tools.spark.transforms.markduplicates.MarkDuplicatesSpark) AbstractMarkDuplicatesTester(org.broadinstitute.hellbender.utils.test.testers.AbstractMarkDuplicatesTester) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Assert(org.testng.Assert) Map(java.util.Map) FileReader(java.io.FileReader) AbstractMarkDuplicatesCommandLineProgramTest(org.broadinstitute.hellbender.utils.test.testers.AbstractMarkDuplicatesCommandLineProgramTest) DuplicationMetrics(org.broadinstitute.hellbender.utils.read.markduplicates.DuplicationMetrics) FileNotFoundException(java.io.FileNotFoundException) ArgumentsBuilder(org.broadinstitute.hellbender.utils.test.ArgumentsBuilder) FileReader(java.io.FileReader) ReadsDataSource(org.broadinstitute.hellbender.engine.ReadsDataSource) MetricsFile(htsjdk.samtools.metrics.MetricsFile) File(java.io.File) MarkDuplicatesIntegrationTest(org.broadinstitute.hellbender.tools.picard.sam.markduplicates.MarkDuplicatesIntegrationTest) Test(org.testng.annotations.Test) AbstractMarkDuplicatesCommandLineProgramTest(org.broadinstitute.hellbender.utils.test.testers.AbstractMarkDuplicatesCommandLineProgramTest)

Aggregations

MetricsFile (htsjdk.samtools.metrics.MetricsFile)32 FileReader (java.io.FileReader)21 File (java.io.File)19 Test (org.testng.annotations.Test)15 BaseTest (org.broadinstitute.hellbender.utils.test.BaseTest)14 SAMRecord (htsjdk.samtools.SAMRecord)10 CommandLineProgramTest (org.broadinstitute.hellbender.CommandLineProgramTest)9 SamReader (htsjdk.samtools.SamReader)6 FileNotFoundException (java.io.FileNotFoundException)5 UserException (org.broadinstitute.hellbender.exceptions.UserException)5 IntervalList (htsjdk.samtools.util.IntervalList)4 Transition (org.broadinstitute.hellbender.tools.picard.analysis.artifacts.Transition)4 ProgressLogger (org.broadinstitute.hellbender.utils.runtime.ProgressLogger)4 InsertSizeMetrics (picard.analysis.InsertSizeMetrics)3 CigarDetailMetrics (gridss.analysis.CigarDetailMetrics)2 IdsvMetrics (gridss.analysis.IdsvMetrics)2 MapqMetrics (gridss.analysis.MapqMetrics)2 SAMFileWriter (htsjdk.samtools.SAMFileWriter)2 SAMFileWriterFactory (htsjdk.samtools.SAMFileWriterFactory)2 SAMReadGroupRecord (htsjdk.samtools.SAMReadGroupRecord)2