Examples with StageOutput com.hartwig.pipeline.StageOutput used on opensource projects

Example 1 with StageOutput

use of com.hartwig.pipeline.StageOutput in project pipeline5 by hartwigmedical.

the class StagedOutputPublisher method publish.

public void publish(final PipelineState state, final SomaticRunMetadata metadata) {
    if (state.status() != PipelineStatus.FAILED && run.isPresent()) {
        List<AddDatatype> addDatatypes = state.stageOutputs().stream().map(StageOutput::datatypes).flatMap(List::stream).collect(Collectors.toList());
        SampleSet set = setResolver.resolve(metadata.set(), useOnlyDBSets);
        Optional<String> tumorSampleName = metadata.maybeTumor().map(SingleSampleRunMetadata::sampleName);
        Optional<String> refSampleName = metadata.maybeReference().map(SingleSampleRunMetadata::sampleName);
        ImmutableAnalysis.Builder alignedReadsAnalysis = eventBuilder(Type.ALIGNMENT);
        ImmutableAnalysis.Builder somaticAnalysis = eventBuilder(Type.SOMATIC);
        ImmutableAnalysis.Builder germlineAnalysis = eventBuilder(Type.GERMLINE);
        OutputIterator.from(blob -> {
            Optional<AddDatatype> dataType = addDatatypes.stream().filter(d -> blob.getName().endsWith(d.path())).findFirst();
            Blob blobWithMd5 = sourceBucket.get(blob.getName());
            if (isSecondary(blobWithMd5)) {
                alignedReadsAnalysis.addOutput(createBlob(tumorSampleName, refSampleName, dataType, blobWithMd5));
            } else {
                if (isGermline(blobWithMd5)) {
                    germlineAnalysis.addOutput(createBlob(tumorSampleName, refSampleName, dataType, blobWithMd5));
                } else if (notSecondary(blobWithMd5)) {
                    somaticAnalysis.addOutput(createBlob(tumorSampleName, refSampleName, dataType, blobWithMd5));
                }
            }
        }, sourceBucket).iterate(metadata);
        publish(PipelineComplete.builder().pipeline(ImmutablePipeline.builder().sample(tumorSampleName.orElseGet(() -> refSampleName.orElseThrow())).bucket(sourceBucket.getName()).runId(run.get().getId()).setId(set.getId()).context(context).addAnalyses(alignedReadsAnalysis.build(), somaticAnalysis.build(), germlineAnalysis.build()).version(Versions.pipelineMajorMinorVersion()).build()).build());
    }
}

Example 2 with StageOutput

use of com.hartwig.pipeline.StageOutput in project pipeline5 by hartwigmedical.

the class StageRunnerTest method skipsStageWhenDisabled.

@Test
public void skipsStageWhenDisabled() {
    when(startingPoint.usePersisted(NAMESPACE)).thenReturn(false);
    final StageOutput skippedOutput = mock(StageOutput.class);
    when(stage.skippedOutput(METADATA)).thenReturn(skippedOutput);
    when(stage.shouldRun(ARGUMENTS)).thenReturn(false);
    assertThat(victim.run(METADATA, stage)).isEqualTo(skippedOutput);
}

Example 3 with StageOutput

use of com.hartwig.pipeline.StageOutput in project pipeline5 by hartwigmedical.

the class VmExecutionLogSummary method ofFailedStages.

public static void ofFailedStages(final Storage storage, final PipelineState state) {
    List<StageOutput> failures = state.stageOutputs().stream().filter(stageOutput -> stageOutput.status().equals(PipelineStatus.FAILED)).collect(Collectors.toList());
    if (!failures.isEmpty()) {
        LOGGER.error("Failures in pipeline stages. Printing each failures full run.log here");
        for (StageOutput failure : failures) {
            for (Blob log : failure.failedLogLocations().stream().map(GoogleStorageLocation::asBlobId).map(storage::get).collect(Collectors.toList())) {
                LOGGER.error("========================================== start {} ==========================================", log.getName());
                LOGGER.error(new String(log.getContent()));
                LOGGER.error("========================================== end {} ==========================================", log.getName());
            }
        }
    }
}

Example 4 with StageOutput

use of com.hartwig.pipeline.StageOutput in project pipeline5 by hartwigmedical.

the class StagedOutputPublisherTest method usesDatatypeAndBarcodeWhenFileMatched.

@Test
public void usesDatatypeAndBarcodeWhenFileMatched() throws Exception {
    when(state.status()).thenReturn(PipelineStatus.SUCCESS);
    String path = TestInputs.referenceSample() + "/germline_caller/reference.germline.vcf.gz";
    Blob vcf = withBucketAndMd5(blob(path));
    Page<Blob> page = pageOf(vcf);
    StageOutput stageOutput = mock(StageOutput.class);
    when(stageOutput.datatypes()).thenReturn(List.of(new AddDatatype(DataType.GERMLINE_VARIANTS, "barcode", new ArchivePath(Folder.from(TestInputs.referenceRunMetadata()), "germline_caller", "reference.germline.vcf.gz"))));
    when(state.stageOutputs()).thenReturn(List.of(stageOutput));
    ArgumentCaptor<PubsubMessage> published = publish(page, TestInputs.defaultSingleSampleRunMetadata());
    PipelineComplete result = OBJECT_MAPPER.readValue(new String(published.getValue().getData().toByteArray()), PipelineComplete.class);
    assertThat(result.pipeline().analyses().get(2).output().get(0).datatype()).hasValue("GERMLINE_VARIANTS");
    assertThat(result.pipeline().analyses().get(2).output().get(0).barcode()).hasValue("barcode");
}

Example 5 with StageOutput

use of com.hartwig.pipeline.StageOutput in project pipeline5 by hartwigmedical.

the class StageRunnerTest method usesPersistedOutputWhenEarlierThanStartingPoint.

@Test
public void usesPersistedOutputWhenEarlierThanStartingPoint() {
    when(startingPoint.usePersisted(NAMESPACE)).thenReturn(true);
    final StageOutput persistedOutput = mock(StageOutput.class);
    when(stage.persistedOutput(METADATA)).thenReturn(persistedOutput);
    when(stage.shouldRun(ARGUMENTS)).thenReturn(true);
    when(stage.tumorReferenceCommands(METADATA)).thenReturn(simpleBash());
    assertThat(victim.run(METADATA, stage)).isEqualTo(persistedOutput);
}