Search in sources :

Example 1 with Arguments

use of com.hartwig.pipeline.Arguments in project pipeline5 by hartwigmedical.

the class SmokeTest method runFullPipelineAndCheckFinalStatus.

public void runFullPipelineAndCheckFinalStatus(final String inputMode, final PipelineStatus expectedStatus) throws Exception {
    PipelineMain victim = new PipelineMain();
    String version = version();
    String setName = noDots(inputMode + "-" + version);
    final String fixtureDir = "smoke_test/" + inputMode + "/";
    final String randomRunId = noDots(RandomStringUtils.random(5, true, false));
    final ImmutableArguments.Builder builder = Arguments.defaultsBuilder(Arguments.DefaultsProfile.DEVELOPMENT.toString()).sampleJson(Resources.testResource(fixtureDir + "samples.json")).cloudSdkPath(CLOUD_SDK_PATH).setId(setName).runId(randomRunId).runGermlineCaller(false).cleanup(true).outputBucket("smoketest-pipeline-output-pilot-1").context(Context.DIAGNOSTIC);
    final String username = System.getProperty("user.name");
    if (username.equals("root")) {
        String privateKeyPath = workingDir() + "/google-key.json";
        builder.privateKeyPath(privateKeyPath).uploadPrivateKeyPath(privateKeyPath);
    } else {
        builder.cloudSdkPath(String.format("/Users/%s/google-cloud-sdk/bin", username));
    }
    Arguments arguments = builder.build();
    Storage storage = StorageProvider.from(arguments, CredentialProvider.from(arguments).get()).get();
    cleanupBucket(inputMode, arguments.outputBucket(), storage);
    PipelineState state = victim.start(arguments);
    assertThat(state.status()).isEqualTo(expectedStatus);
    File expectedFilesResource = new File(Resources.testResource(fixtureDir + "expected_output_files"));
    List<String> expectedFiles = FileUtils.readLines(expectedFilesResource, FILE_ENCODING);
    final String outputDir = setName + "-" + randomRunId;
    List<String> actualFiles = listOutput(outputDir, arguments.outputBucket(), storage);
    assertThat(actualFiles).containsOnlyElementsOf(expectedFiles);
    cleanupBucket(outputDir, arguments.outputBucket(), storage);
}
Also used : PipelineState(com.hartwig.pipeline.PipelineState) PipelineMain(com.hartwig.pipeline.PipelineMain) Storage(com.google.cloud.storage.Storage) Arguments(com.hartwig.pipeline.Arguments) ImmutableArguments(com.hartwig.pipeline.ImmutableArguments) ImmutableArguments(com.hartwig.pipeline.ImmutableArguments) File(java.io.File)

Example 2 with Arguments

use of com.hartwig.pipeline.Arguments in project pipeline5 by hartwigmedical.

the class BwaAligner method run.

public AlignmentOutput run(final SingleSampleRunMetadata metadata) throws Exception {
    StageTrace trace = new StageTrace(NAMESPACE, metadata.sampleName(), StageTrace.ExecutorType.COMPUTE_ENGINE).start();
    RuntimeBucket rootBucket = RuntimeBucket.from(storage, NAMESPACE, metadata, arguments, labels);
    Sample sample = sampleSource.sample(metadata);
    if (sample.bam().isPresent()) {
        String noPrefix = sample.bam().orElseThrow().replace("gs://", "");
        int firstSlash = noPrefix.indexOf("/");
        String bucket = noPrefix.substring(0, firstSlash);
        String path = noPrefix.substring(firstSlash + 1);
        return AlignmentOutput.builder().sample(metadata.sampleName()).status(PipelineStatus.PROVIDED).maybeAlignments(GoogleStorageLocation.of(bucket, path)).build();
    }
    final ResourceFiles resourceFiles = buildResourceFiles(arguments);
    sampleUpload.run(sample, rootBucket);
    List<Future<PipelineStatus>> futures = new ArrayList<>();
    List<GoogleStorageLocation> perLaneBams = new ArrayList<>();
    List<ReportComponent> laneLogComponents = new ArrayList<>();
    List<GoogleStorageLocation> laneFailedLogs = new ArrayList<>();
    for (Lane lane : sample.lanes()) {
        RuntimeBucket laneBucket = RuntimeBucket.from(storage, laneNamespace(lane), metadata, arguments, labels);
        BashStartupScript bash = BashStartupScript.of(laneBucket.name());
        InputDownload first = new InputDownload(GoogleStorageLocation.of(rootBucket.name(), fastQFileName(sample.name(), lane.firstOfPairPath())));
        InputDownload second = new InputDownload(GoogleStorageLocation.of(rootBucket.name(), fastQFileName(sample.name(), lane.secondOfPairPath())));
        bash.addCommand(first).addCommand(second);
        bash.addCommands(OverrideReferenceGenomeCommand.overrides(arguments));
        SubStageInputOutput alignment = new LaneAlignment(arguments.sbpApiRunId().isPresent(), resourceFiles.refGenomeFile(), first.getLocalTargetPath(), second.getLocalTargetPath(), metadata.sampleName(), lane).apply(SubStageInputOutput.empty(metadata.sampleName()));
        perLaneBams.add(GoogleStorageLocation.of(laneBucket.name(), resultsDirectory.path(alignment.outputFile().fileName())));
        bash.addCommands(alignment.bash()).addCommand(new OutputUpload(GoogleStorageLocation.of(laneBucket.name(), resultsDirectory.path()), RuntimeFiles.typical()));
        futures.add(executorService.submit(() -> runWithRetries(metadata, laneBucket, VirtualMachineJobDefinition.alignment(laneId(lane).toLowerCase(), bash, resultsDirectory))));
        laneLogComponents.add(new RunLogComponent(laneBucket, laneNamespace(lane), Folder.from(metadata), resultsDirectory));
        laneFailedLogs.add(GoogleStorageLocation.of(laneBucket.name(), RunLogComponent.LOG_FILE));
    }
    AlignmentOutput output;
    if (lanesSuccessfullyComplete(futures)) {
        List<InputDownload> laneBams = perLaneBams.stream().map(InputDownload::new).collect(Collectors.toList());
        BashStartupScript mergeMarkdupsBash = BashStartupScript.of(rootBucket.name());
        laneBams.forEach(mergeMarkdupsBash::addCommand);
        SubStageInputOutput merged = new MergeMarkDups(laneBams.stream().map(InputDownload::getLocalTargetPath).filter(path -> path.endsWith("bam")).collect(Collectors.toList())).apply(SubStageInputOutput.empty(metadata.sampleName()));
        mergeMarkdupsBash.addCommands(merged.bash());
        mergeMarkdupsBash.addCommand(new OutputUpload(GoogleStorageLocation.of(rootBucket.name(), resultsDirectory.path()), RuntimeFiles.typical()));
        PipelineStatus status = runWithRetries(metadata, rootBucket, VirtualMachineJobDefinition.mergeMarkdups(mergeMarkdupsBash, resultsDirectory));
        ImmutableAlignmentOutput.Builder outputBuilder = AlignmentOutput.builder().sample(metadata.sampleName()).status(status).maybeAlignments(GoogleStorageLocation.of(rootBucket.name(), resultsDirectory.path(merged.outputFile().fileName()))).addAllReportComponents(laneLogComponents).addAllFailedLogLocations(laneFailedLogs).addFailedLogLocations(GoogleStorageLocation.of(rootBucket.name(), RunLogComponent.LOG_FILE)).addReportComponents(new RunLogComponent(rootBucket, Aligner.NAMESPACE, Folder.from(metadata), resultsDirectory));
        if (!arguments.outputCram()) {
            outputBuilder.addReportComponents(new SingleFileComponent(rootBucket, Aligner.NAMESPACE, Folder.from(metadata), bam(metadata.sampleName()), bam(metadata.sampleName()), resultsDirectory), new SingleFileComponent(rootBucket, Aligner.NAMESPACE, Folder.from(metadata), bai(bam(metadata.sampleName())), bai(bam(metadata.sampleName())), resultsDirectory)).addDatatypes(new AddDatatype(DataType.ALIGNED_READS, metadata.barcode(), new ArchivePath(Folder.from(metadata), BwaAligner.NAMESPACE, bam(metadata.sampleName()))), new AddDatatype(DataType.ALIGNED_READS_INDEX, metadata.barcode(), new ArchivePath(Folder.from(metadata), BwaAligner.NAMESPACE, bai(metadata.sampleName()))));
        }
        output = outputBuilder.build();
    } else {
        output = AlignmentOutput.builder().sample(metadata.sampleName()).status(PipelineStatus.FAILED).build();
    }
    trace.stop();
    executorService.shutdown();
    return output;
}
Also used : Arguments(com.hartwig.pipeline.Arguments) StageTrace(com.hartwig.pipeline.trace.StageTrace) SubStageInputOutput(com.hartwig.pipeline.stages.SubStageInputOutput) Aligner(com.hartwig.pipeline.alignment.Aligner) InputDownload(com.hartwig.pipeline.execution.vm.InputDownload) ArrayList(java.util.ArrayList) VirtualMachineJobDefinition(com.hartwig.pipeline.execution.vm.VirtualMachineJobDefinition) Future(java.util.concurrent.Future) RuntimeBucket(com.hartwig.pipeline.storage.RuntimeBucket) PipelineStatus(com.hartwig.pipeline.execution.PipelineStatus) ExecutorService(java.util.concurrent.ExecutorService) BashStartupScript(com.hartwig.pipeline.execution.vm.BashStartupScript) DataType(com.hartwig.pipeline.datatypes.DataType) FileTypes.bai(com.hartwig.pipeline.datatypes.FileTypes.bai) ImmutableAlignmentOutput(com.hartwig.pipeline.alignment.ImmutableAlignmentOutput) GoogleStorageLocation(com.hartwig.pipeline.storage.GoogleStorageLocation) Lane(com.hartwig.patient.Lane) ArchivePath(com.hartwig.pipeline.metadata.ArchivePath) SampleUpload(com.hartwig.pipeline.storage.SampleUpload) Folder(com.hartwig.pipeline.report.Folder) ResultsDirectory(com.hartwig.pipeline.ResultsDirectory) OutputUpload(com.hartwig.pipeline.execution.vm.OutputUpload) DefaultBackoffPolicy(com.hartwig.pipeline.failsafe.DefaultBackoffPolicy) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) File(java.io.File) SingleFileComponent(com.hartwig.pipeline.report.SingleFileComponent) Failsafe(net.jodah.failsafe.Failsafe) ResourceFilesFactory.buildResourceFiles(com.hartwig.pipeline.resource.ResourceFilesFactory.buildResourceFiles) ExecutionException(java.util.concurrent.ExecutionException) List(java.util.List) Sample(com.hartwig.patient.Sample) AddDatatype(com.hartwig.pipeline.metadata.AddDatatype) AlignmentOutput(com.hartwig.pipeline.alignment.AlignmentOutput) OverrideReferenceGenomeCommand(com.hartwig.pipeline.resource.OverrideReferenceGenomeCommand) RuntimeFiles(com.hartwig.pipeline.execution.vm.RuntimeFiles) SingleSampleRunMetadata(com.hartwig.pipeline.metadata.SingleSampleRunMetadata) Storage(com.google.cloud.storage.Storage) Labels(com.hartwig.pipeline.labels.Labels) FileTypes.bam(com.hartwig.pipeline.datatypes.FileTypes.bam) ResourceFiles(com.hartwig.pipeline.resource.ResourceFiles) ComputeEngine(com.hartwig.pipeline.execution.vm.ComputeEngine) ReportComponent(com.hartwig.pipeline.report.ReportComponent) RunLogComponent(com.hartwig.pipeline.report.RunLogComponent) SampleSource(com.hartwig.pipeline.alignment.sample.SampleSource) RunLogComponent(com.hartwig.pipeline.report.RunLogComponent) PipelineStatus(com.hartwig.pipeline.execution.PipelineStatus) ArrayList(java.util.ArrayList) ReportComponent(com.hartwig.pipeline.report.ReportComponent) AddDatatype(com.hartwig.pipeline.metadata.AddDatatype) ArchivePath(com.hartwig.pipeline.metadata.ArchivePath) BashStartupScript(com.hartwig.pipeline.execution.vm.BashStartupScript) InputDownload(com.hartwig.pipeline.execution.vm.InputDownload) SingleFileComponent(com.hartwig.pipeline.report.SingleFileComponent) Sample(com.hartwig.patient.Sample) Lane(com.hartwig.patient.Lane) SubStageInputOutput(com.hartwig.pipeline.stages.SubStageInputOutput) ImmutableAlignmentOutput(com.hartwig.pipeline.alignment.ImmutableAlignmentOutput) StageTrace(com.hartwig.pipeline.trace.StageTrace) ResourceFilesFactory.buildResourceFiles(com.hartwig.pipeline.resource.ResourceFilesFactory.buildResourceFiles) ResourceFiles(com.hartwig.pipeline.resource.ResourceFiles) OutputUpload(com.hartwig.pipeline.execution.vm.OutputUpload) ImmutableAlignmentOutput(com.hartwig.pipeline.alignment.ImmutableAlignmentOutput) AlignmentOutput(com.hartwig.pipeline.alignment.AlignmentOutput) RuntimeBucket(com.hartwig.pipeline.storage.RuntimeBucket) Future(java.util.concurrent.Future) GoogleStorageLocation(com.hartwig.pipeline.storage.GoogleStorageLocation)

Example 3 with Arguments

use of com.hartwig.pipeline.Arguments in project pipeline5 by hartwigmedical.

the class GoogleComputeEngineTest method stopsInstanceWithPersistentDisksUponFailure.

@Test
public void stopsInstanceWithPersistentDisksUponFailure() {
    Arguments arguments = Arguments.testDefaultsBuilder().useLocalSsds(false).build();
    victim = new GoogleComputeEngine(arguments, compute, z -> {
    }, lifecycleManager, bucketWatcher, mock(Labels.class));
    returnFailed();
    victim.submit(runtimeBucket.getRuntimeBucket(), jobDefinition);
    verify(lifecycleManager).stop(FIRST_ZONE_NAME, INSTANCE_NAME);
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) Instance(com.google.api.services.compute.model.Instance) InstanceList(com.google.api.services.compute.model.InstanceList) Tags(com.google.api.services.compute.model.Tags) Arguments(com.hartwig.pipeline.Arguments) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) MockRuntimeBucket(com.hartwig.pipeline.testsupport.MockRuntimeBucket) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Zone(com.google.api.services.compute.model.Zone) AttachedDisk(com.google.api.services.compute.model.AttachedDisk) Lists(com.google.common.collect.Lists) ArgumentCaptor(org.mockito.ArgumentCaptor) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) TestInputs(com.hartwig.pipeline.testsupport.TestInputs) Map(java.util.Map) PipelineStatus(com.hartwig.pipeline.execution.PipelineStatus) Before(org.junit.Before) ZoneList(com.google.api.services.compute.model.ZoneList) State(com.hartwig.pipeline.execution.vm.BucketCompletionWatcher.State) ResultsDirectory(com.hartwig.pipeline.ResultsDirectory) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) NetworkInterface(com.google.api.services.compute.model.NetworkInterface) Scheduling(com.google.api.services.compute.model.Scheduling) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Operation(com.google.api.services.compute.model.Operation) Mockito.verify(org.mockito.Mockito.verify) Mockito.never(org.mockito.Mockito.never) List(java.util.List) Image(com.google.api.services.compute.model.Image) Labels(com.hartwig.pipeline.labels.Labels) Collections(java.util.Collections) Compute(com.google.api.services.compute.Compute) Mockito.mock(org.mockito.Mockito.mock) Arguments(com.hartwig.pipeline.Arguments) Test(org.junit.Test)

Example 4 with Arguments

use of com.hartwig.pipeline.Arguments in project pipeline5 by hartwigmedical.

the class GoogleComputeEngineTest method disablesStartupScriptWhenInstanceWithPersistentDisksFailsRemotely.

@Test
public void disablesStartupScriptWhenInstanceWithPersistentDisksFailsRemotely() throws Exception {
    Arguments arguments = Arguments.testDefaultsBuilder().useLocalSsds(false).build();
    victim = new GoogleComputeEngine(arguments, compute, z -> {
    }, lifecycleManager, bucketWatcher, mock(Labels.class));
    returnFailed();
    victim.submit(runtimeBucket.getRuntimeBucket(), jobDefinition);
    verify(lifecycleManager).disableStartupScript(FIRST_ZONE_NAME, INSTANCE_NAME);
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) Instance(com.google.api.services.compute.model.Instance) InstanceList(com.google.api.services.compute.model.InstanceList) Tags(com.google.api.services.compute.model.Tags) Arguments(com.hartwig.pipeline.Arguments) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) MockRuntimeBucket(com.hartwig.pipeline.testsupport.MockRuntimeBucket) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Zone(com.google.api.services.compute.model.Zone) AttachedDisk(com.google.api.services.compute.model.AttachedDisk) Lists(com.google.common.collect.Lists) ArgumentCaptor(org.mockito.ArgumentCaptor) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) TestInputs(com.hartwig.pipeline.testsupport.TestInputs) Map(java.util.Map) PipelineStatus(com.hartwig.pipeline.execution.PipelineStatus) Before(org.junit.Before) ZoneList(com.google.api.services.compute.model.ZoneList) State(com.hartwig.pipeline.execution.vm.BucketCompletionWatcher.State) ResultsDirectory(com.hartwig.pipeline.ResultsDirectory) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) NetworkInterface(com.google.api.services.compute.model.NetworkInterface) Scheduling(com.google.api.services.compute.model.Scheduling) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Operation(com.google.api.services.compute.model.Operation) Mockito.verify(org.mockito.Mockito.verify) Mockito.never(org.mockito.Mockito.never) List(java.util.List) Image(com.google.api.services.compute.model.Image) Labels(com.hartwig.pipeline.labels.Labels) Collections(java.util.Collections) Compute(com.google.api.services.compute.Compute) Mockito.mock(org.mockito.Mockito.mock) Arguments(com.hartwig.pipeline.Arguments) Test(org.junit.Test)

Example 5 with Arguments

use of com.hartwig.pipeline.Arguments in project pipeline5 by hartwigmedical.

the class RuntimeBucketTest method appliesLabels.

@Test
public void appliesLabels() {
    Arguments arguments = Arguments.testDefaultsBuilder().costCenterLabel("development").userLabel("username").build();
    RuntimeBucket.from(storage, NAMESPACE, referenceRunMetadata(), arguments, Labels.of(arguments, TestInputs.defaultSomaticRunMetadata()));
    assertThat(bucketInfo.getValue().getLabels()).isEqualTo(Map.of("cost_center", "development", "run_id", "test", "sample", "tumor", "user", "username"));
}
Also used : Arguments(com.hartwig.pipeline.Arguments) Test(org.junit.Test)

Aggregations

Arguments (com.hartwig.pipeline.Arguments)9 Test (org.junit.Test)5 Storage (com.google.cloud.storage.Storage)4 ResultsDirectory (com.hartwig.pipeline.ResultsDirectory)4 PipelineStatus (com.hartwig.pipeline.execution.PipelineStatus)4 Labels (com.hartwig.pipeline.labels.Labels)4 List (java.util.List)4 Before (org.junit.Before)4 Compute (com.google.api.services.compute.Compute)3 AttachedDisk (com.google.api.services.compute.model.AttachedDisk)3 Image (com.google.api.services.compute.model.Image)3 Instance (com.google.api.services.compute.model.Instance)3 InstanceList (com.google.api.services.compute.model.InstanceList)3 NetworkInterface (com.google.api.services.compute.model.NetworkInterface)3 Operation (com.google.api.services.compute.model.Operation)3 Scheduling (com.google.api.services.compute.model.Scheduling)3 Tags (com.google.api.services.compute.model.Tags)3 Zone (com.google.api.services.compute.model.Zone)3 ZoneList (com.google.api.services.compute.model.ZoneList)3 Lists (com.google.common.collect.Lists)3