Search in sources :

Example 1 with ProcessException

use of com.milaboratory.cli.ProcessException in project mixcr by milaboratory.

the class ActionAlign method go.

@Override
@SuppressWarnings("unchecked")
public void go(ActionHelper helper) throws Exception {
    // FIXME remove in 2.2
    if (actionParameters.printNonFunctionalWarnings())
        System.out.println("WARNING: -wf / --non-functional-warnings option is deprecated, will be removed in 2.2 " + "release. Use --verbose instead.");
    // Saving initial timestamp
    long beginTimestamp = System.currentTimeMillis();
    // Getting aligner parameters
    VDJCAlignerParameters alignerParameters = actionParameters.getAlignerParameters();
    // FIXME remove in 2.3
    if (actionParameters.getSaveOriginalReads()) {
        System.out.println("WARNING: -g / --save-reads option is deprecated, will be removed in 2.3 " + "release. Use -OsaveOriginalReads=true.");
        alignerParameters.setSaveOriginalReads(true);
    }
    // FIXME remove in 2.3
    if (actionParameters.getSaveReadDescription()) {
        System.out.println("WARNING: -a / --save-description option is deprecated, will be removed in 2.3 " + "release. Use -OsaveOriginalReads=true.");
        alignerParameters.setSaveOriginalReads(true);
    }
    if (!actionParameters.overrides.isEmpty()) {
        // Perform parameters overriding
        alignerParameters = JsonOverrider.override(alignerParameters, VDJCAlignerParameters.class, actionParameters.overrides);
        if (alignerParameters == null)
            throw new ProcessException("Failed to override some parameter.");
    }
    // FIXME remove in 2.2
    if (actionParameters.allowDifferentVJLoci != null && actionParameters.allowDifferentVJLoci) {
        System.out.println("Warning: usage of --diff-loci is deprecated. Use -OallowChimeras=true instead.");
        alignerParameters.setAllowChimeras(true);
    }
    // Creating aligner
    VDJCAligner aligner = VDJCAligner.createAligner(alignerParameters, actionParameters.isInputPaired(), !actionParameters.getNoMerge());
    // Detect if automatic featureToAlign correction is required
    int totalV = 0, totalVErrors = 0, hasVRegion = 0;
    GeneFeature correctingFeature = alignerParameters.getVAlignerParameters().getGeneFeatureToAlign().hasReversedRegions() ? GeneFeature.VRegionWithP : GeneFeature.VRegion;
    VDJCLibrary library = VDJCLibraryRegistry.getDefault().getLibrary(actionParameters.library, actionParameters.species);
    System.out.println("Reference library: " + library.getLibraryId());
    // Printing library level warnings, if specified for the library
    if (!library.getWarnings().isEmpty()) {
        System.out.println("Library warnings:");
        for (String l : library.getWarnings()) System.out.println(l);
    }
    // Printing citation notice, if specified for the library
    if (!library.getCitations().isEmpty()) {
        System.out.println("Please cite:");
        for (String l : library.getCitations()) System.out.println(l);
    }
    for (VDJCGene gene : library.getGenes(actionParameters.getChains())) {
        if (gene.getGeneType() == GeneType.Variable) {
            totalV++;
            if (!alignerParameters.containsRequiredFeature(gene)) {
                totalVErrors++;
                if (gene.getPartitioning().isAvailable(correctingFeature))
                    hasVRegion++;
            }
        }
    }
    // Performing V featureToAlign correction if needed
    if (totalVErrors > totalV * 0.9 && hasVRegion > totalVErrors * 0.8) {
        System.out.println("WARNING: forcing -OvParameters.geneFeatureToAlign=" + GeneFeature.encode(correctingFeature) + " since current gene feature (" + GeneFeature.encode(alignerParameters.getVAlignerParameters().getGeneFeatureToAlign()) + ") is absent in " + Util.PERCENT_FORMAT.format(100.0 * totalVErrors / totalV) + "% of V genes.");
        alignerParameters.getVAlignerParameters().setGeneFeatureToAlign(correctingFeature);
    }
    int numberOfExcludedNFGenes = 0;
    int numberOfExcludedFGenes = 0;
    for (VDJCGene gene : library.getGenes(actionParameters.getChains())) {
        NucleotideSequence featureSequence = alignerParameters.extractFeatureToAlign(gene);
        // exclusionReason is null ==> gene is not excluded
        String exclusionReason = null;
        if (featureSequence == null)
            exclusionReason = "absent " + GeneFeature.encode(alignerParameters.getFeatureToAlign(gene.getGeneType()));
        else if (featureSequence.containsWildcards())
            exclusionReason = "wildcard symbols in " + GeneFeature.encode(alignerParameters.getFeatureToAlign(gene.getGeneType()));
        if (exclusionReason == null)
            // If there are no reasons to exclude the gene, adding it to aligner
            aligner.addGene(gene);
        else {
            if (gene.isFunctional()) {
                ++numberOfExcludedFGenes;
                if (actionParameters.verbose())
                    System.out.println("WARNING: Functional gene " + gene.getName() + " excluded due to " + exclusionReason);
            } else
                ++numberOfExcludedNFGenes;
        }
    }
    if (actionParameters.printWarnings() && numberOfExcludedFGenes > 0)
        System.out.println("WARNING: " + numberOfExcludedFGenes + " functional genes were excluded, re-run " + "with --verbose option to see the list of excluded genes and exclusion reason.");
    if (actionParameters.verbose() && numberOfExcludedNFGenes > 0)
        System.out.println("WARNING: " + numberOfExcludedNFGenes + " non-functional genes excluded.");
    if (aligner.getVGenesToAlign().isEmpty())
        throw new ProcessException("No V genes to align. Aborting execution. See warnings for more info " + "(turn on verbose warnings by adding --verbose option).");
    if (aligner.getJGenesToAlign().isEmpty())
        throw new ProcessException("No J genes to align. Aborting execution. See warnings for more info " + "(turn on verbose warnings by adding --verbose option).");
    AlignerReport report = new AlignerReport();
    report.setStartMillis(beginTimestamp);
    report.setInputFiles(actionParameters.getInputsForReport());
    report.setOutputFiles(actionParameters.getOutputsForReport());
    report.setCommandLine(helper.getCommandLineArguments());
    // Attaching report to aligner
    aligner.setEventsListener(report);
    try (SequenceReaderCloseable<? extends SequenceRead> reader = actionParameters.createReader();
        VDJCAlignmentsWriter writer = actionParameters.getOutputName().equals(".") ? null : new VDJCAlignmentsWriter(actionParameters.getOutputName());
        SequenceWriter notAlignedWriter = actionParameters.failedReadsR1 == null ? null : (actionParameters.isInputPaired() ? new PairedFastqWriter(actionParameters.failedReadsR1, actionParameters.failedReadsR2) : new SingleFastqWriter(actionParameters.failedReadsR1))) {
        if (writer != null)
            writer.header(aligner);
        OutputPort<? extends SequenceRead> sReads = reader;
        CanReportProgress progress = (CanReportProgress) reader;
        if (actionParameters.limit != 0) {
            sReads = new CountLimitingOutputPort<>(sReads, actionParameters.limit);
            progress = SmartProgressReporter.extractProgress((CountLimitingOutputPort<?>) sReads);
        }
        final boolean writeAllResults = actionParameters.getWriteAllResults();
        EnumMap<GeneType, VDJCHit[]> emptyHits = new EnumMap<>(GeneType.class);
        for (GeneType gt : GeneType.values()) if (alignerParameters.getGeneAlignerParameters(gt) != null)
            emptyHits.put(gt, new VDJCHit[0]);
        final PairedEndReadsLayout readsLayout = alignerParameters.getReadsLayout();
        SmartProgressReporter.startProgressReport("Alignment", progress);
        OutputPort<Chunk<? extends SequenceRead>> mainInputReads = CUtils.buffered((OutputPort) chunked(sReads, 64), 16);
        OutputPort<VDJCAlignmentResult> alignments = unchunked(new ParallelProcessor(mainInputReads, chunked(aligner), actionParameters.threads));
        for (VDJCAlignmentResult result : CUtils.it(new OrderedOutputPort<>(alignments, new Indexer<VDJCAlignmentResult>() {

            @Override
            public long getIndex(VDJCAlignmentResult o) {
                return o.read.getId();
            }
        }))) {
            VDJCAlignments alignment = result.alignment;
            SequenceRead read = result.read;
            if (alignment == null) {
                if (writeAllResults) // Creating empty alignment object if alignment for current read failed
                {
                    Target target = readsLayout.createTargets(read)[0];
                    alignment = new VDJCAlignments(emptyHits, target.targets, SequenceHistory.RawSequence.of(read.getId(), target), alignerParameters.isSaveOriginalReads() ? new SequenceRead[] { read } : null);
                } else {
                    if (notAlignedWriter != null)
                        notAlignedWriter.write(result.read);
                    continue;
                }
            }
            if (alignment.isChimera())
                report.onChimera();
            if (writer != null)
                writer.write(alignment);
        }
        if (writer != null)
            writer.setNumberOfProcessedReads(reader.getNumberOfReads());
    }
    report.setFinishMillis(System.currentTimeMillis());
    // Writing report to stout
    System.out.println("============= Report ==============");
    Util.writeReportToStdout(report);
    if (actionParameters.report != null)
        Util.writeReport(actionParameters.report, report);
    if (actionParameters.jsonReport != null)
        Util.writeJsonReport(actionParameters.jsonReport, report);
}
Also used : VDJCAlignerParameters(com.milaboratory.mixcr.vdjaligners.VDJCAlignerParameters) VDJCAlignmentsWriter(com.milaboratory.mixcr.basictypes.VDJCAlignmentsWriter) VDJCAligner(com.milaboratory.mixcr.vdjaligners.VDJCAligner) ParallelProcessor(cc.redberry.pipe.blocks.ParallelProcessor) Target(com.milaboratory.core.Target) Indexer(cc.redberry.pipe.util.Indexer) CanReportProgress(com.milaboratory.util.CanReportProgress) SingleFastqWriter(com.milaboratory.core.io.sequence.fastq.SingleFastqWriter) VDJCAlignments(com.milaboratory.mixcr.basictypes.VDJCAlignments) PairedEndReadsLayout(com.milaboratory.core.PairedEndReadsLayout) VDJCAlignmentResult(com.milaboratory.mixcr.vdjaligners.VDJCAlignmentResult) PairedFastqWriter(com.milaboratory.core.io.sequence.fastq.PairedFastqWriter) Chunk(cc.redberry.pipe.util.Chunk) ProcessException(com.milaboratory.cli.ProcessException) CountLimitingOutputPort(cc.redberry.pipe.util.CountLimitingOutputPort) SequenceWriter(com.milaboratory.core.io.sequence.SequenceWriter) NucleotideSequence(com.milaboratory.core.sequence.NucleotideSequence) SequenceRead(com.milaboratory.core.io.sequence.SequenceRead)

Example 2 with ProcessException

use of com.milaboratory.cli.ProcessException in project mixcr by milaboratory.

the class ActionAssembleContigs method go.

@Override
public void go(ActionHelper helper) throws Exception {
    // TODO FIX!!!!!!!!!!!!!
    if (parameters.threads > 1)
        throw new ParameterException("Multithreaded processing is not supported yet.");
    long beginTimestamp = System.currentTimeMillis();
    FullSeqAssemblerParameters p = FullSeqAssemblerParameters.getByName("default");
    if (!parameters.overrides.isEmpty()) {
        // Perform parameters overriding
        p = JsonOverrider.override(p, FullSeqAssemblerParameters.class, parameters.overrides);
        if (p == null)
            throw new ProcessException("Failed to override some parameter.");
    }
    final FullSeqAssemblerReport report = new FullSeqAssemblerReport();
    FullSeqAssemblerParameters assemblerParameters = p;
    int totalClonesCount = 0;
    List<VDJCGene> genes;
    VDJCAlignerParameters alignerParameters;
    CloneAssemblerParameters cloneAssemblerParameters;
    try (ClnAReader reader = new ClnAReader(parameters.getInputFileName(), VDJCLibraryRegistry.getDefault());
        PrimitivO tmpOut = new PrimitivO(new BufferedOutputStream(new FileOutputStream(parameters.getOutputFileName())))) {
        final CloneFactory cloneFactory = new CloneFactory(reader.getAssemblerParameters().getCloneFactoryParameters(), reader.getAssemblingFeatures(), reader.getGenes(), reader.getAlignerParameters().getFeaturesToAlignMap());
        alignerParameters = reader.getAlignerParameters();
        cloneAssemblerParameters = reader.getAssemblerParameters();
        genes = reader.getGenes();
        IOUtil.registerGeneReferences(tmpOut, genes, alignerParameters);
        ClnAReader.CloneAlignmentsPort cloneAlignmentsPort = reader.clonesAndAlignments();
        SmartProgressReporter.startProgressReport("Assembling", cloneAlignmentsPort);
        OutputPort<Clone[]> parallelProcessor = new ParallelProcessor<>(cloneAlignmentsPort, cloneAlignments -> {
            FullSeqAssembler fullSeqAssembler = new FullSeqAssembler(cloneFactory, assemblerParameters, cloneAlignments.clone, alignerParameters);
            fullSeqAssembler.setReport(report);
            FullSeqAssembler.RawVariantsData rawVariantsData = fullSeqAssembler.calculateRawData(() -> {
                try {
                    return cloneAlignments.alignments();
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            });
            return fullSeqAssembler.callVariants(rawVariantsData);
        }, parameters.threads);
        for (Clone[] clones : CUtils.it(parallelProcessor)) {
            totalClonesCount += clones.length;
            for (Clone cl : clones) tmpOut.writeObject(cl);
        }
        assert report.getInitialCloneCount() == reader.numberOfClones();
    }
    assert report.getFinalCloneCount() == totalClonesCount;
    assert report.getFinalCloneCount() >= report.getInitialCloneCount();
    Clone[] clones = new Clone[totalClonesCount];
    try (PrimitivI tmpIn = new PrimitivI(new BufferedInputStream(new FileInputStream(parameters.getOutputFileName())))) {
        IOUtil.registerGeneReferences(tmpIn, genes, alignerParameters);
        int i = 0;
        for (Clone clone : CUtils.it(new PipeDataInputReader<>(Clone.class, tmpIn, totalClonesCount))) clones[i++] = clone;
    }
    Arrays.sort(clones, Comparator.comparingDouble(c -> -c.getCount()));
    for (int i = 0; i < clones.length; i++) clones[i] = clones[i].setId(i);
    CloneSet cloneSet = new CloneSet(Arrays.asList(clones), genes, alignerParameters.getFeaturesToAlignMap(), alignerParameters, cloneAssemblerParameters);
    try (CloneSetIO.CloneSetWriter writer = new CloneSetIO.CloneSetWriter(cloneSet, parameters.getOutputFileName())) {
        SmartProgressReporter.startProgressReport(writer);
        writer.write();
    }
    ReportWrapper reportWrapper = new ReportWrapper(command(), report);
    reportWrapper.setStartMillis(beginTimestamp);
    reportWrapper.setInputFiles(parameters.getInputFileName());
    reportWrapper.setOutputFiles(parameters.getOutputFileName());
    reportWrapper.setCommandLine(helper.getCommandLineArguments());
    reportWrapper.setFinishMillis(System.currentTimeMillis());
    // Writing report to stout
    System.out.println("============= Report ==============");
    Util.writeReportToStdout(report);
    if (parameters.report != null)
        Util.writeReport(parameters.report, reportWrapper);
    if (parameters.jsonReport != null)
        Util.writeJsonReport(parameters.jsonReport, reportWrapper);
}
Also used : Parameters(com.beust.jcommander.Parameters) java.util(java.util) ParameterException(com.beust.jcommander.ParameterException) Action(com.milaboratory.cli.Action) Parameter(com.beust.jcommander.Parameter) CloneAssemblerParameters(com.milaboratory.mixcr.assembler.CloneAssemblerParameters) com.milaboratory.mixcr.basictypes(com.milaboratory.mixcr.basictypes) FullSeqAssemblerParameters(com.milaboratory.mixcr.assembler.fullseq.FullSeqAssemblerParameters) ParallelProcessor(cc.redberry.pipe.blocks.ParallelProcessor) VDJCAlignerParameters(com.milaboratory.mixcr.vdjaligners.VDJCAlignerParameters) PositiveInteger(com.beust.jcommander.validators.PositiveInteger) CloneFactory(com.milaboratory.mixcr.assembler.CloneFactory) FullSeqAssembler(com.milaboratory.mixcr.assembler.fullseq.FullSeqAssembler) ActionHelper(com.milaboratory.cli.ActionHelper) PrimitivO(com.milaboratory.primitivio.PrimitivO) CUtils(cc.redberry.pipe.CUtils) FullSeqAssemblerReport(com.milaboratory.mixcr.assembler.fullseq.FullSeqAssemblerReport) PrimitivI(com.milaboratory.primitivio.PrimitivI) OutputPort(cc.redberry.pipe.OutputPort) PipeDataInputReader(com.milaboratory.primitivio.PipeDataInputReader) java.io(java.io) VDJCGene(io.repseq.core.VDJCGene) SmartProgressReporter(com.milaboratory.util.SmartProgressReporter) DynamicParameter(com.beust.jcommander.DynamicParameter) VDJCLibraryRegistry(io.repseq.core.VDJCLibraryRegistry) ProcessException(com.milaboratory.cli.ProcessException) ActionParametersWithOutput(com.milaboratory.cli.ActionParametersWithOutput) VDJCAlignerParameters(com.milaboratory.mixcr.vdjaligners.VDJCAlignerParameters) FullSeqAssemblerParameters(com.milaboratory.mixcr.assembler.fullseq.FullSeqAssemblerParameters) CloneAssemblerParameters(com.milaboratory.mixcr.assembler.CloneAssemblerParameters) ParallelProcessor(cc.redberry.pipe.blocks.ParallelProcessor) ParameterException(com.beust.jcommander.ParameterException) PrimitivO(com.milaboratory.primitivio.PrimitivO) FullSeqAssembler(com.milaboratory.mixcr.assembler.fullseq.FullSeqAssembler) FullSeqAssemblerReport(com.milaboratory.mixcr.assembler.fullseq.FullSeqAssemblerReport) ProcessException(com.milaboratory.cli.ProcessException) VDJCGene(io.repseq.core.VDJCGene) CloneFactory(com.milaboratory.mixcr.assembler.CloneFactory) PrimitivI(com.milaboratory.primitivio.PrimitivI)

Aggregations

ParallelProcessor (cc.redberry.pipe.blocks.ParallelProcessor)2 ProcessException (com.milaboratory.cli.ProcessException)2 VDJCAlignerParameters (com.milaboratory.mixcr.vdjaligners.VDJCAlignerParameters)2 CUtils (cc.redberry.pipe.CUtils)1 OutputPort (cc.redberry.pipe.OutputPort)1 Chunk (cc.redberry.pipe.util.Chunk)1 CountLimitingOutputPort (cc.redberry.pipe.util.CountLimitingOutputPort)1 Indexer (cc.redberry.pipe.util.Indexer)1 DynamicParameter (com.beust.jcommander.DynamicParameter)1 Parameter (com.beust.jcommander.Parameter)1 ParameterException (com.beust.jcommander.ParameterException)1 Parameters (com.beust.jcommander.Parameters)1 PositiveInteger (com.beust.jcommander.validators.PositiveInteger)1 Action (com.milaboratory.cli.Action)1 ActionHelper (com.milaboratory.cli.ActionHelper)1 ActionParametersWithOutput (com.milaboratory.cli.ActionParametersWithOutput)1 PairedEndReadsLayout (com.milaboratory.core.PairedEndReadsLayout)1 Target (com.milaboratory.core.Target)1 SequenceRead (com.milaboratory.core.io.sequence.SequenceRead)1 SequenceWriter (com.milaboratory.core.io.sequence.SequenceWriter)1