Examples with OutputPort cc.redberry.pipe.OutputPort used on opensource projects

Example 1 with OutputPort

use of cc.redberry.pipe.OutputPort in project mixcr by milaboratory.

the class ActionAssembleContigs method go.

@Override
public void go(ActionHelper helper) throws Exception {
    // TODO FIX!!!!!!!!!!!!!
    if (parameters.threads > 1)
        throw new ParameterException("Multithreaded processing is not supported yet.");
    long beginTimestamp = System.currentTimeMillis();
    FullSeqAssemblerParameters p = FullSeqAssemblerParameters.getByName("default");
    if (!parameters.overrides.isEmpty()) {
        // Perform parameters overriding
        p = JsonOverrider.override(p, FullSeqAssemblerParameters.class, parameters.overrides);
        if (p == null)
            throw new ProcessException("Failed to override some parameter.");
    }
    final FullSeqAssemblerReport report = new FullSeqAssemblerReport();
    FullSeqAssemblerParameters assemblerParameters = p;
    int totalClonesCount = 0;
    List<VDJCGene> genes;
    VDJCAlignerParameters alignerParameters;
    CloneAssemblerParameters cloneAssemblerParameters;
    try (ClnAReader reader = new ClnAReader(parameters.getInputFileName(), VDJCLibraryRegistry.getDefault());
        PrimitivO tmpOut = new PrimitivO(new BufferedOutputStream(new FileOutputStream(parameters.getOutputFileName())))) {
        final CloneFactory cloneFactory = new CloneFactory(reader.getAssemblerParameters().getCloneFactoryParameters(), reader.getAssemblingFeatures(), reader.getGenes(), reader.getAlignerParameters().getFeaturesToAlignMap());
        alignerParameters = reader.getAlignerParameters();
        cloneAssemblerParameters = reader.getAssemblerParameters();
        genes = reader.getGenes();
        IOUtil.registerGeneReferences(tmpOut, genes, alignerParameters);
        ClnAReader.CloneAlignmentsPort cloneAlignmentsPort = reader.clonesAndAlignments();
        SmartProgressReporter.startProgressReport("Assembling", cloneAlignmentsPort);
        OutputPort<Clone[]> parallelProcessor = new ParallelProcessor<>(cloneAlignmentsPort, cloneAlignments -> {
            FullSeqAssembler fullSeqAssembler = new FullSeqAssembler(cloneFactory, assemblerParameters, cloneAlignments.clone, alignerParameters);
            fullSeqAssembler.setReport(report);
            FullSeqAssembler.RawVariantsData rawVariantsData = fullSeqAssembler.calculateRawData(() -> {
                try {
                    return cloneAlignments.alignments();
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            });
            return fullSeqAssembler.callVariants(rawVariantsData);
        }, parameters.threads);
        for (Clone[] clones : CUtils.it(parallelProcessor)) {
            totalClonesCount += clones.length;
            for (Clone cl : clones) tmpOut.writeObject(cl);
        }
        assert report.getInitialCloneCount() == reader.numberOfClones();
    }
    assert report.getFinalCloneCount() == totalClonesCount;
    assert report.getFinalCloneCount() >= report.getInitialCloneCount();
    Clone[] clones = new Clone[totalClonesCount];
    try (PrimitivI tmpIn = new PrimitivI(new BufferedInputStream(new FileInputStream(parameters.getOutputFileName())))) {
        IOUtil.registerGeneReferences(tmpIn, genes, alignerParameters);
        int i = 0;
        for (Clone clone : CUtils.it(new PipeDataInputReader<>(Clone.class, tmpIn, totalClonesCount))) clones[i++] = clone;
    }
    Arrays.sort(clones, Comparator.comparingDouble(c -> -c.getCount()));
    for (int i = 0; i < clones.length; i++) clones[i] = clones[i].setId(i);
    CloneSet cloneSet = new CloneSet(Arrays.asList(clones), genes, alignerParameters.getFeaturesToAlignMap(), alignerParameters, cloneAssemblerParameters);
    try (CloneSetIO.CloneSetWriter writer = new CloneSetIO.CloneSetWriter(cloneSet, parameters.getOutputFileName())) {
        SmartProgressReporter.startProgressReport(writer);
        writer.write();
    }
    ReportWrapper reportWrapper = new ReportWrapper(command(), report);
    reportWrapper.setStartMillis(beginTimestamp);
    reportWrapper.setInputFiles(parameters.getInputFileName());
    reportWrapper.setOutputFiles(parameters.getOutputFileName());
    reportWrapper.setCommandLine(helper.getCommandLineArguments());
    reportWrapper.setFinishMillis(System.currentTimeMillis());
    // Writing report to stout
    System.out.println("============= Report ==============");
    Util.writeReportToStdout(report);
    if (parameters.report != null)
        Util.writeReport(parameters.report, reportWrapper);
    if (parameters.jsonReport != null)
        Util.writeJsonReport(parameters.jsonReport, reportWrapper);
}

Example 2 with OutputPort

use of cc.redberry.pipe.OutputPort in project mixcr by milaboratory.

the class FullSeqAssembler method calculateRawData.

public RawVariantsData calculateRawData(Supplier<OutputPort<VDJCAlignments>> alignments) {
    if (!sequenceToVariantId.isEmpty())
        throw new IllegalStateException();
    for (byte letter = 0; letter < NucleotideSequence.ALPHABET.basicSize(); letter++) {
        NucleotideSequence seq = new NucleotideSequence(new byte[] { letter });
        sequenceToVariantId.put(seq, letter);
        variantIdToSequence.put(letter, seq);
    }
    sequenceToVariantId.put(NucleotideSequence.EMPTY, NucleotideSequence.ALPHABET.basicSize());
    variantIdToSequence.put(NucleotideSequence.ALPHABET.basicSize(), NucleotideSequence.EMPTY);
    TIntIntHashMap coverage = new TIntIntHashMap();
    TIntObjectHashMap<TIntObjectHashMap<VariantAggregator>> variants = new TIntObjectHashMap<>();
    int nAlignments = 0;
    for (VDJCAlignments al : CUtils.it(alignments.get())) {
        ++nAlignments;
        for (PointSequence point : toPointSequences(al)) {
            int seqIndex = getVariantIndex(point.sequence.getSequence());
            coverage.adjustOrPutValue(point.point, 1, 1);
            TIntObjectHashMap<VariantAggregator> map = variants.get(point.point);
            if (map == null)
                variants.put(point.point, map = new TIntObjectHashMap<>());
            VariantAggregator var = map.get(seqIndex);
            if (var == null)
                map.put(point.point, var = new VariantAggregator());
            var.count += 1;
            var.sumQuality += 0x7F & point.quality;
        }
    }
    assert nAlignments > 0;
    long[] forSort = new long[coverage.size()];
    TIntIntIterator iterator = coverage.iterator();
    int i = 0;
    while (iterator.hasNext()) {
        iterator.advance();
        forSort[i++] = -((((long) iterator.value()) << 32) | iterator.key());
    }
    Arrays.sort(forSort);
    int[] pointsArray = Arrays.stream(forSort).mapToInt(l -> (int) (-l)).toArray();
    TIntIntHashMap revIndex = new TIntIntHashMap();
    for (int j = 0; j < pointsArray.length; j++) revIndex.put(pointsArray[j], j);
    int[] coverageArray = Arrays.stream(forSort).mapToInt(l -> (int) ((-l) >> 32)).toArray();
    int[][] packedData = new int[pointsArray.length][nAlignments];
    for (int[] aPackedData : packedData) Arrays.fill(aPackedData, -1);
    i = 0;
    for (VDJCAlignments al : CUtils.it(alignments.get())) {
        for (PointSequence point : toPointSequences(al)) {
            int pointIndex = revIndex.get(point.point);
            packedData[pointIndex][i] = (sequenceToVariantId.get(point.sequence.getSequence()) << 8) | (0xFF & point.quality);
        }
        i++;
    }
    return new RawVariantsData(nAlignments, pointsArray, coverageArray) {

        @Override
        OutputPort<int[]> createPort() {
            return CUtils.asOutputPort(Arrays.asList(packedData));
        }
    };
}

Example 3 with OutputPort

use of cc.redberry.pipe.OutputPort in project mixcr by milaboratory.

the class ActionSlice method sliceClnA.

void sliceClnA(ActionHelper helper) throws Exception {
    try (ClnAReader reader = new ClnAReader(params.getInputFileName(), VDJCLibraryRegistry.getDefault());
        ClnAWriter writer = new ClnAWriter(params.getOutputFileName())) {
        // Getting full clone set
        CloneSet cloneSet = reader.readCloneSet();
        // old clone id -> new clone id
        TIntIntHashMap idMapping = new TIntIntHashMap();
        long newNumberOfAlignments = 0;
        // Creating new cloneset
        List<Clone> clones = new ArrayList<>();
        int i = 0;
        List<OutputPort<VDJCAlignments>> allAlignmentsList = new ArrayList<>();
        for (Long cloneId_ : params.ids) {
            int cloneId = (int) ((long) cloneId_);
            newNumberOfAlignments += reader.numberOfAlignmentsInClone(cloneId);
            Clone clone = cloneSet.get(cloneId);
            idMapping.put(clone.getId(), i);
            clones.add(clone.setId(i).resetParentCloneSet());
            OutputPort<VDJCAlignments> als = reader.readAlignmentsOfClone(cloneId);
            final int ii = i;
            allAlignmentsList.add(() -> {
                VDJCAlignments al = als.take();
                if (al == null)
                    return null;
                return al.updateCloneIndex(ii);
            });
            i++;
        }
        CloneSet newCloneSet = new CloneSet(clones, cloneSet.getUsedGenes(), cloneSet.getAlignedFeatures(), cloneSet.getAlignmentParameters(), cloneSet.getAssemblerParameters());
        OutputPort<VDJCAlignments> allAlignmentsPortRaw = new FlatteningOutputPort<>(CUtils.asOutputPort(allAlignmentsList));
        AtomicLong idGen = new AtomicLong();
        OutputPort<VDJCAlignments> allAlignmentsPort = () -> {
            VDJCAlignments al = allAlignmentsPortRaw.take();
            if (al == null)
                return null;
            return al.setAlignmentsIndex(idGen.getAndIncrement());
        };
        writer.writeClones(newCloneSet);
        writer.sortAlignments(allAlignmentsPort, newNumberOfAlignments);
        writer.writeAlignmentsAndIndex();
    }
}