Examples with AsyncBufferedIterator au.edu.wehi.idsv.util.AsyncBufferedIterator used on opensource projects

Example 6 with AsyncBufferedIterator

use of au.edu.wehi.idsv.util.AsyncBufferedIterator in project gridss by PapenfussLab.

the class SplitReadRealigner method createSupplementaryAlignmentFastq.

protected int createSupplementaryAlignmentFastq(File input, File fq, boolean isRecursive) throws IOException {
    int recordsWritten = 0;
    try (SamReader reader = readerFactory.open(input)) {
        try (AsyncBufferedIterator<SAMRecord> bufferedIt = new AsyncBufferedIterator<>(reader.iterator(), input.getName())) {
            try (FastqWriter writer = fastqWriterFactory.newWriter(fq)) {
                SplitReadFastqExtractionIterator fastqit = new SplitReadFastqExtractionIterator(bufferedIt, isRecursive, minSoftClipLength, minSoftClipQuality, !isRecursive && isProcessSecondaryAlignments(), eidgen);
                while (fastqit.hasNext()) {
                    writer.write(fastqit.next());
                    recordsWritten++;
                }
            }
        }
    }
    return recordsWritten;
}

Example 7 with AsyncBufferedIterator

use of au.edu.wehi.idsv.util.AsyncBufferedIterator in project gridss by PapenfussLab.

the class SplitReadRealigner method mergeSupplementaryAlignment.

private void mergeSupplementaryAlignment(File input, List<File> aligned, File output) throws IOException {
    log.info("Merging split read alignments for ", output);
    File suppMerged = FileSystemContext.getWorkingFileFor(output, "gridss.tmp.SplitReadAligner.sa.");
    File tmpoutput = FileSystemContext.getWorkingFileFor(output);
    tmpFiles.add(suppMerged);
    tmpFiles.add(tmpoutput);
    List<SamReader> suppReaders = new ArrayList<>();
    List<PeekingIterator<SAMRecord>> suppIt = new ArrayList<>();
    SAMFileHeader header;
    try (SamReader reader = readerFactory.open(input)) {
        header = reader.getFileHeader();
        for (File sf : aligned) {
            SamReader suppReader = readerFactory.open(sf);
            suppReaders.add(suppReader);
            suppIt.add(new AsyncBufferedIterator<>(new NmTagIterator(suppReader.iterator(), pc.getReference()), sf.getName()));
        }
        try (SAMFileWriter inputWriter = writerFactory.makeSAMOrBAMWriter(header, true, tmpoutput)) {
            try (SAMFileWriter suppWriter = writerFactory.makeSAMOrBAMWriter(header, false, suppMerged)) {
                try (AsyncBufferedIterator<SAMRecord> bufferedIt = new AsyncBufferedIterator<>(new NmTagIterator(reader.iterator(), pc.getReference()), input.getName())) {
                    mergeSupplementaryAlignment(bufferedIt, suppIt, inputWriter, suppWriter);
                }
            }
        }
    } finally {
        for (Iterator<SAMRecord> it : suppIt) {
            CloserUtil.close(it);
        }
        for (SamReader sr : suppReaders) {
            sr.close();
        }
    }
    if (header.getSortOrder() != null && header.getSortOrder() != SortOrder.unsorted) {
        File suppMergedsorted = FileSystemContext.getWorkingFileFor(output, "gridss.tmp.SplitReadAligner.sorted.sa.");
        tmpFiles.add(suppMergedsorted);
        SAMFileUtil.sort(pc.getFileSystemContext(), suppMerged, suppMergedsorted, header.getSortOrder());
        FileHelper.move(suppMergedsorted, suppMerged, true);
    }
    SAMFileUtil.merge(ImmutableList.of(tmpoutput, suppMerged), output);
}

Example 8 with AsyncBufferedIterator

use of au.edu.wehi.idsv.util.AsyncBufferedIterator in project gridss by PapenfussLab.

the class ReadsToBedpe method doWork.

@Override
protected int doWork() {
    log.debug("Setting language-neutral locale");
    java.util.Locale.setDefault(Locale.ROOT);
    validateParameters();
    SamReaderFactory readerFactory = SamReaderFactory.make();
    try {
        try (SamReader reader = readerFactory.open(INPUT)) {
            SAMFileHeader header = reader.getFileHeader();
            SAMSequenceDictionary dict = header.getSequenceDictionary();
            // ExecutorService threadpool = Executors.newFixedThreadPool(WORKER_THREADS, new ThreadFactoryBuilder().setDaemon(false).setNameFormat("Worker-%d").build());
            try (CloseableIterator<SAMRecord> rawit = new AsyncBufferedIterator<SAMRecord>(reader.iterator(), 3, 64)) {
                ProgressLoggingSAMRecordIterator logit = new ProgressLoggingSAMRecordIterator(rawit, new ProgressLogger(log));
                // ParallelTransformIterator<SAMRecord, List<String>> it = new ParallelTransformIterator<>(logit, r -> asBedPe(dict, r), 16 + 2 * WORKER_THREADS, threadpool);
                Iterator<List<String>> it = Iterators.transform(logit, r -> asBedPe(dict, r));
                int i = 0;
                try (BufferedWriter writer = new BufferedWriter(new FileWriter(OUTPUT))) {
                    while (it.hasNext()) {
                        for (String line : it.next()) {
                            if (line != null) {
                                writer.write(line);
                                writer.write('\n');
                            }
                        }
                        i++;
                    }
                    if (i % 1000 == 0) {
                        writer.flush();
                    }
                }
            }
        }
    } catch (IOException e) {
        log.error(e);
        return -1;
    }
    return 0;
}

Example 9 with AsyncBufferedIterator

use of au.edu.wehi.idsv.util.AsyncBufferedIterator in project gridss by PapenfussLab.

the class SAMFileUtil method merge.

/**
 * Merges a set of SAM files into a single file.
 * The SAM header is taken from the first input file.
 * @param input input files. All input files must have the same sort order.
 * @param output output file.
 * @param readerFactory
 * @param writerFactory
 * @throws IOException
 */
public static void merge(Collection<File> input, File output, SamReaderFactory readerFactory, SAMFileWriterFactory writerFactory) throws IOException {
    if (input == null)
        throw new IllegalArgumentException("input is null");
    File tmpFile = gridss.Defaults.OUTPUT_TO_TEMP_FILE ? FileSystemContext.getWorkingFileFor(output, "gridss.tmp.merging.SAMFileUtil.") : output;
    Map<SamReader, AsyncBufferedIterator<SAMRecord>> map = new HashMap<>(input.size());
    SAMFileHeader header = null;
    try {
        for (File in : input) {
            SamReader r = readerFactory.open(in);
            SAMFileHeader currentHeader = r.getFileHeader();
            if (header == null) {
                header = currentHeader;
            }
            if (header.getSortOrder() != null && currentHeader.getSortOrder() != null && header.getSortOrder() != currentHeader.getSortOrder()) {
                throw new IllegalArgumentException(String.format("Sort order %s of %s does not match %s of %s", currentHeader.getSortOrder(), input, header.getSortOrder(), input.iterator().next()));
            }
            map.put(r, new AsyncBufferedIterator<>(r.iterator(), in.getName()));
        }
        try (SAMFileWriter writer = writerFactory.makeSAMOrBAMWriter(header, true, tmpFile)) {
            Queue<PeekingIterator<SAMRecord>> queue = createMergeQueue(header.getSortOrder());
            for (PeekingIterator<SAMRecord> it : map.values()) {
                if (it.hasNext()) {
                    queue.add(it);
                }
            }
            while (!queue.isEmpty()) {
                PeekingIterator<SAMRecord> it = queue.poll();
                SAMRecord r = it.next();
                writer.addAlignment(r);
                if (it.hasNext()) {
                    queue.add(it);
                }
            }
        }
        for (Entry<SamReader, AsyncBufferedIterator<SAMRecord>> entry : map.entrySet()) {
            CloserUtil.close(entry.getValue());
            CloserUtil.close(entry.getKey());
        }
        if (tmpFile != output) {
            FileHelper.move(tmpFile, output, true);
        }
    } finally {
        for (Entry<SamReader, AsyncBufferedIterator<SAMRecord>> entry : map.entrySet()) {
            CloserUtil.close(entry.getValue());
            CloserUtil.close(entry.getKey());
        }
        if (tmpFile != output & tmpFile.exists()) {
            FileHelper.delete(tmpFile, true);
        }
    }
}