Examples with PersistedDataset com.hartwig.pipeline.reruns.PersistedDataset used on opensource projects

Example 1 with PersistedDataset

use of com.hartwig.pipeline.reruns.PersistedDataset in project pipeline5 by hartwigmedical.

the class PersistedAlignmentTest method returnsPersistedCramsWhenExists.

@Test
public void returnsPersistedCramsWhenExists() {
    PersistedDataset persistedDataset = mock(PersistedDataset.class);
    when(persistedDataset.path(TestInputs.referenceRunMetadata().sampleName(), DataType.ALIGNED_READS)).thenReturn(Optional.of(PERSISTED_REFERENCE_CRAM));
    PersistedAlignment victim = new PersistedAlignment(persistedDataset, Arguments.testDefaults());
    AlignmentOutput output = victim.run(TestInputs.referenceRunMetadata());
    assertThat(output.sample()).isEqualTo("reference");
    assertThat(output.status()).isEqualTo(PipelineStatus.PERSISTED);
    assertThat(output.alignments()).isEqualTo(PERSISTED_REFERENCE_CRAM);
}

Example 2 with PersistedDataset

use of com.hartwig.pipeline.reruns.PersistedDataset in project pipeline5 by hartwigmedical.

the class PipelineMain method start.

public PipelineState start(final Arguments arguments) {
    LOGGER.info("Arguments are [{}]", arguments);
    Versions.printAll();
    try {
        GoogleCredentials credentials = CredentialProvider.from(arguments).get();
        Storage storage = StorageProvider.from(arguments, credentials).get();
        Publisher turquoisePublisher = PublisherProvider.from(arguments, credentials).get("turquoise.events");
        Publisher pipelinePublisher = PublisherProvider.from(arguments, credentials).get(PipelineComplete.TOPIC);
        SomaticMetadataApi somaticMetadataApi = SomaticMetadataApiProvider.from(arguments, storage, pipelinePublisher).get();
        SingleSampleEventListener referenceEventListener = new SingleSampleEventListener();
        SingleSampleEventListener tumorEventListener = new SingleSampleEventListener();
        SomaticRunMetadata somaticRunMetadata = somaticMetadataApi.get();
        InputMode mode = new ModeResolver().apply(somaticRunMetadata);
        LOGGER.info("Starting pipeline in [{}] mode", mode);
        String ini = somaticRunMetadata.isSingleSample() ? "single_sample" : arguments.shallow() ? "shallow" : "somatic";
        PipelineProperties eventSubjects = PipelineProperties.builder().sample(somaticRunMetadata.maybeTumor().map(SingleSampleRunMetadata::sampleName).orElseGet(() -> somaticRunMetadata.reference().sampleName())).runId(arguments.sbpApiRunId()).set(somaticRunMetadata.set()).referenceBarcode(somaticRunMetadata.maybeReference().map(SingleSampleRunMetadata::barcode)).tumorBarcode(somaticRunMetadata.maybeTumor().map(SingleSampleRunMetadata::barcode)).type(ini).build();
        somaticMetadataApi.start();
        startedEvent(eventSubjects, turquoisePublisher, arguments.publishToTurquoise());
        BlockingQueue<BamMetricsOutput> referenceBamMetricsOutputQueue = new ArrayBlockingQueue<>(1);
        BlockingQueue<BamMetricsOutput> tumorBamMetricsOutputQueue = new ArrayBlockingQueue<>(1);
        BlockingQueue<FlagstatOutput> referenceFlagstatOutputQueue = new ArrayBlockingQueue<>(1);
        BlockingQueue<FlagstatOutput> tumorFlagstatOutputQueue = new ArrayBlockingQueue<>(1);
        BlockingQueue<GermlineCallerOutput> germlineCallerOutputQueue = new ArrayBlockingQueue<>(1);
        StartingPoint startingPoint = new StartingPoint(arguments);
        PersistedDataset persistedDataset = arguments.biopsy().<PersistedDataset>map(b -> new ApiPersistedDataset(SbpRestApi.newInstance(arguments.sbpApiUrl()), ObjectMappers.get(), b, arguments.project())).orElse(new NoopPersistedDataset());
        PipelineState state = new FullPipeline(singleSamplePipeline(arguments, credentials, storage, referenceEventListener, somaticRunMetadata, referenceBamMetricsOutputQueue, germlineCallerOutputQueue, referenceFlagstatOutputQueue, startingPoint, persistedDataset, mode), singleSamplePipeline(arguments, credentials, storage, tumorEventListener, somaticRunMetadata, tumorBamMetricsOutputQueue, germlineCallerOutputQueue, tumorFlagstatOutputQueue, startingPoint, persistedDataset, mode), somaticPipeline(arguments, credentials, storage, somaticRunMetadata, referenceBamMetricsOutputQueue, tumorBamMetricsOutputQueue, referenceFlagstatOutputQueue, tumorFlagstatOutputQueue, startingPoint, persistedDataset, mode), Executors.newCachedThreadPool(), referenceEventListener, tumorEventListener, somaticMetadataApi, CleanupProvider.from(arguments, storage).get()).run();
        completedEvent(eventSubjects, turquoisePublisher, state.status().toString(), arguments.publishToTurquoise());
        VmExecutionLogSummary.ofFailedStages(storage, state);
        return state;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 3 with PersistedDataset

use of com.hartwig.pipeline.reruns.PersistedDataset in project pipeline5 by hartwigmedical.

the class SomaticPipeline method run.

public PipelineState run(final AlignmentPair pair) {
    PipelineState state = new PipelineState();
    LOGGER.info("Pipeline5 somatic pipeline starting for set [{}]", metadata.set());
    final ResourceFiles resourceFiles = buildResourceFiles(arguments);
    try {
        Future<AmberOutput> amberOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new Amber(pair, resourceFiles, persistedDataset, arguments)));
        Future<CobaltOutput> cobaltOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new Cobalt(pair, resourceFiles, persistedDataset, arguments)));
        Future<SageOutput> sageSomaticOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new SageSomaticCaller(pair, persistedDataset, resourceFiles, arguments)));
        Future<SageOutput> sageGermlineOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new SageGermlineCaller(pair, persistedDataset, resourceFiles)));
        Future<GridssOutput> structuralCallerOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new Gridss(pair, resourceFiles, persistedDataset)));
        SageOutput sageSomaticOutput = pipelineResults.add(state.add(sageSomaticOutputFuture.get()));
        SageOutput sageGermlineOutput = pipelineResults.add(state.add(sageGermlineOutputFuture.get()));
        AmberOutput amberOutput = pipelineResults.add(state.add(amberOutputFuture.get()));
        CobaltOutput cobaltOutput = pipelineResults.add(state.add(cobaltOutputFuture.get()));
        GridssOutput structuralCallerOutput = pipelineResults.add(state.add(structuralCallerOutputFuture.get()));
        if (state.shouldProceed()) {
            Future<PaveOutput> paveSomaticOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new PaveSomatic(resourceFiles, sageSomaticOutput, persistedDataset)));
            Future<PaveOutput> paveGermlineOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new PaveGermline(resourceFiles, sageGermlineOutput, persistedDataset)));
            PaveOutput paveSomaticOutput = pipelineResults.add(state.add(paveSomaticOutputFuture.get()));
            PaveOutput paveGermlineOutput = pipelineResults.add(state.add(paveGermlineOutputFuture.get()));
            Future<GripssOutput> gripssSomaticOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new GripssSomatic(structuralCallerOutput, persistedDataset, resourceFiles)));
            Future<GripssOutput> gripssGermlineOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new GripssGermline(structuralCallerOutput, persistedDataset, resourceFiles)));
            GripssOutput gripssSomaticProcessOutput = pipelineResults.add(state.add(gripssSomaticOutputFuture.get()));
            GripssOutput gripssGermlineProcessOutput = pipelineResults.add(state.add(gripssGermlineOutputFuture.get()));
            if (state.shouldProceed()) {
                Future<PurpleOutput> purpleOutputFuture = executorService.submit(() -> pipelineResults.add(state.add(stageRunner.run(metadata, new Purple(resourceFiles, paveSomaticOutput, paveGermlineOutput, metadata.maybeTumor().map(t -> gripssSomaticProcessOutput).orElse(gripssGermlineProcessOutput), amberOutput, cobaltOutput, persistedDataset, arguments)))));
                PurpleOutput purpleOutput = purpleOutputFuture.get();
                if (state.shouldProceed()) {
                    BamMetricsOutput tumorMetrics = metadata.maybeTumor().map(t -> pollOrThrow(tumorBamMetricsOutputQueue, "tumor metrics")).orElse(skippedMetrics(metadata.sampleName()));
                    BamMetricsOutput referenceMetrics = metadata.maybeReference().map(t -> pollOrThrow(referenceBamMetricsOutputQueue, "reference metrics")).orElse(skippedMetrics(metadata.sampleName()));
                    FlagstatOutput tumorFlagstat = metadata.maybeTumor().map(t -> pollOrThrow(tumorFlagstatOutputQueue, "tumor flagstat")).orElse(skippedFlagstat(metadata.sampleName()));
                    FlagstatOutput referenceFlagstat = metadata.maybeReference().map(t -> pollOrThrow(referenceFlagstatOutputQueue, "reference flagstat")).orElse(skippedFlagstat(metadata.sampleName()));
                    Future<VirusOutput> virusOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new VirusAnalysis(pair, resourceFiles, persistedDataset, purpleOutput, tumorMetrics)));
                    Future<HealthCheckOutput> healthCheckOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new HealthChecker(referenceMetrics, tumorMetrics, referenceFlagstat, tumorFlagstat, purpleOutput)));
                    Future<LinxSomaticOutput> linxSomaticOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new LinxSomatic(purpleOutput, resourceFiles, persistedDataset)));
                    Future<LinxGermlineOutput> linxGermlineOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new LinxGermline(gripssGermlineProcessOutput, resourceFiles, persistedDataset)));
                    Future<LilacOutput> lilacOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new Lilac(pair, resourceFiles, purpleOutput)));
                    Future<SigsOutput> signatureOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new Sigs(purpleOutput, resourceFiles)));
                    Future<ChordOutput> chordOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new Chord(arguments.refGenomeVersion(), purpleOutput, persistedDataset)));
                    pipelineResults.add(state.add(healthCheckOutputFuture.get()));
                    LinxSomaticOutput linxSomaticOutput = pipelineResults.add(state.add(linxSomaticOutputFuture.get()));
                    pipelineResults.add(state.add(linxGermlineOutputFuture.get()));
                    pipelineResults.add(state.add(lilacOutputFuture.get()));
                    Future<CuppaOutput> cuppaOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new Cuppa(purpleOutput, linxSomaticOutput, resourceFiles, persistedDataset)));
                    Future<PeachOutput> peachOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new Peach(purpleOutput, resourceFiles, persistedDataset)));
                    VirusOutput virusOutput = pipelineResults.add(state.add(virusOutputFuture.get()));
                    ChordOutput chordOutput = pipelineResults.add(state.add(chordOutputFuture.get()));
                    CuppaOutput cuppaOutput = pipelineResults.add(state.add(cuppaOutputFuture.get()));
                    PeachOutput peachOutput = pipelineResults.add(state.add(peachOutputFuture.get()));
                    ProtectOutput protectOutput = pipelineResults.add(state.add(executorService.submit(() -> stageRunner.run(metadata, new Protect(purpleOutput, linxSomaticOutput, virusOutput, chordOutput, resourceFiles, persistedDataset))).get()));
                    Future<OrangeOutput> orangeOutputFuture = executorService.submit(() -> stageRunner.run(metadata, new Orange(tumorMetrics, referenceMetrics, tumorFlagstat, referenceFlagstat, sageSomaticOutput, sageGermlineOutput, purpleOutput, chordOutput, linxSomaticOutput, cuppaOutput, virusOutput, protectOutput, peachOutput, resourceFiles)));
                    pipelineResults.add(state.add(signatureOutputFuture.get()));
                    pipelineResults.add(state.add(orangeOutputFuture.get()));
                    pipelineResults.compose(metadata, "Somatic");
                }
            }
        }
    } catch (InterruptedException | ExecutionException e) {
        throw new RuntimeException(e);
    }
    return state;
}