use of com.hartwig.pipeline.Arguments in project pipeline5 by hartwigmedical.
the class SmokeTest method runFullPipelineAndCheckFinalStatus.
public void runFullPipelineAndCheckFinalStatus(final String inputMode, final PipelineStatus expectedStatus) throws Exception {
PipelineMain victim = new PipelineMain();
String version = version();
String setName = noDots(inputMode + "-" + version);
final String fixtureDir = "smoke_test/" + inputMode + "/";
final String randomRunId = noDots(RandomStringUtils.random(5, true, false));
final ImmutableArguments.Builder builder = Arguments.defaultsBuilder(Arguments.DefaultsProfile.DEVELOPMENT.toString()).sampleJson(Resources.testResource(fixtureDir + "samples.json")).cloudSdkPath(CLOUD_SDK_PATH).setId(setName).runId(randomRunId).runGermlineCaller(false).cleanup(true).outputBucket("smoketest-pipeline-output-pilot-1").context(Context.DIAGNOSTIC);
final String username = System.getProperty("user.name");
if (username.equals("root")) {
String privateKeyPath = workingDir() + "/google-key.json";
builder.privateKeyPath(privateKeyPath).uploadPrivateKeyPath(privateKeyPath);
} else {
builder.cloudSdkPath(String.format("/Users/%s/google-cloud-sdk/bin", username));
}
Arguments arguments = builder.build();
Storage storage = StorageProvider.from(arguments, CredentialProvider.from(arguments).get()).get();
cleanupBucket(inputMode, arguments.outputBucket(), storage);
PipelineState state = victim.start(arguments);
assertThat(state.status()).isEqualTo(expectedStatus);
File expectedFilesResource = new File(Resources.testResource(fixtureDir + "expected_output_files"));
List<String> expectedFiles = FileUtils.readLines(expectedFilesResource, FILE_ENCODING);
final String outputDir = setName + "-" + randomRunId;
List<String> actualFiles = listOutput(outputDir, arguments.outputBucket(), storage);
assertThat(actualFiles).containsOnlyElementsOf(expectedFiles);
cleanupBucket(outputDir, arguments.outputBucket(), storage);
}
use of com.hartwig.pipeline.Arguments in project pipeline5 by hartwigmedical.
the class BwaAligner method run.
public AlignmentOutput run(final SingleSampleRunMetadata metadata) throws Exception {
StageTrace trace = new StageTrace(NAMESPACE, metadata.sampleName(), StageTrace.ExecutorType.COMPUTE_ENGINE).start();
RuntimeBucket rootBucket = RuntimeBucket.from(storage, NAMESPACE, metadata, arguments, labels);
Sample sample = sampleSource.sample(metadata);
if (sample.bam().isPresent()) {
String noPrefix = sample.bam().orElseThrow().replace("gs://", "");
int firstSlash = noPrefix.indexOf("/");
String bucket = noPrefix.substring(0, firstSlash);
String path = noPrefix.substring(firstSlash + 1);
return AlignmentOutput.builder().sample(metadata.sampleName()).status(PipelineStatus.PROVIDED).maybeAlignments(GoogleStorageLocation.of(bucket, path)).build();
}
final ResourceFiles resourceFiles = buildResourceFiles(arguments);
sampleUpload.run(sample, rootBucket);
List<Future<PipelineStatus>> futures = new ArrayList<>();
List<GoogleStorageLocation> perLaneBams = new ArrayList<>();
List<ReportComponent> laneLogComponents = new ArrayList<>();
List<GoogleStorageLocation> laneFailedLogs = new ArrayList<>();
for (Lane lane : sample.lanes()) {
RuntimeBucket laneBucket = RuntimeBucket.from(storage, laneNamespace(lane), metadata, arguments, labels);
BashStartupScript bash = BashStartupScript.of(laneBucket.name());
InputDownload first = new InputDownload(GoogleStorageLocation.of(rootBucket.name(), fastQFileName(sample.name(), lane.firstOfPairPath())));
InputDownload second = new InputDownload(GoogleStorageLocation.of(rootBucket.name(), fastQFileName(sample.name(), lane.secondOfPairPath())));
bash.addCommand(first).addCommand(second);
bash.addCommands(OverrideReferenceGenomeCommand.overrides(arguments));
SubStageInputOutput alignment = new LaneAlignment(arguments.sbpApiRunId().isPresent(), resourceFiles.refGenomeFile(), first.getLocalTargetPath(), second.getLocalTargetPath(), metadata.sampleName(), lane).apply(SubStageInputOutput.empty(metadata.sampleName()));
perLaneBams.add(GoogleStorageLocation.of(laneBucket.name(), resultsDirectory.path(alignment.outputFile().fileName())));
bash.addCommands(alignment.bash()).addCommand(new OutputUpload(GoogleStorageLocation.of(laneBucket.name(), resultsDirectory.path()), RuntimeFiles.typical()));
futures.add(executorService.submit(() -> runWithRetries(metadata, laneBucket, VirtualMachineJobDefinition.alignment(laneId(lane).toLowerCase(), bash, resultsDirectory))));
laneLogComponents.add(new RunLogComponent(laneBucket, laneNamespace(lane), Folder.from(metadata), resultsDirectory));
laneFailedLogs.add(GoogleStorageLocation.of(laneBucket.name(), RunLogComponent.LOG_FILE));
}
AlignmentOutput output;
if (lanesSuccessfullyComplete(futures)) {
List<InputDownload> laneBams = perLaneBams.stream().map(InputDownload::new).collect(Collectors.toList());
BashStartupScript mergeMarkdupsBash = BashStartupScript.of(rootBucket.name());
laneBams.forEach(mergeMarkdupsBash::addCommand);
SubStageInputOutput merged = new MergeMarkDups(laneBams.stream().map(InputDownload::getLocalTargetPath).filter(path -> path.endsWith("bam")).collect(Collectors.toList())).apply(SubStageInputOutput.empty(metadata.sampleName()));
mergeMarkdupsBash.addCommands(merged.bash());
mergeMarkdupsBash.addCommand(new OutputUpload(GoogleStorageLocation.of(rootBucket.name(), resultsDirectory.path()), RuntimeFiles.typical()));
PipelineStatus status = runWithRetries(metadata, rootBucket, VirtualMachineJobDefinition.mergeMarkdups(mergeMarkdupsBash, resultsDirectory));
ImmutableAlignmentOutput.Builder outputBuilder = AlignmentOutput.builder().sample(metadata.sampleName()).status(status).maybeAlignments(GoogleStorageLocation.of(rootBucket.name(), resultsDirectory.path(merged.outputFile().fileName()))).addAllReportComponents(laneLogComponents).addAllFailedLogLocations(laneFailedLogs).addFailedLogLocations(GoogleStorageLocation.of(rootBucket.name(), RunLogComponent.LOG_FILE)).addReportComponents(new RunLogComponent(rootBucket, Aligner.NAMESPACE, Folder.from(metadata), resultsDirectory));
if (!arguments.outputCram()) {
outputBuilder.addReportComponents(new SingleFileComponent(rootBucket, Aligner.NAMESPACE, Folder.from(metadata), bam(metadata.sampleName()), bam(metadata.sampleName()), resultsDirectory), new SingleFileComponent(rootBucket, Aligner.NAMESPACE, Folder.from(metadata), bai(bam(metadata.sampleName())), bai(bam(metadata.sampleName())), resultsDirectory)).addDatatypes(new AddDatatype(DataType.ALIGNED_READS, metadata.barcode(), new ArchivePath(Folder.from(metadata), BwaAligner.NAMESPACE, bam(metadata.sampleName()))), new AddDatatype(DataType.ALIGNED_READS_INDEX, metadata.barcode(), new ArchivePath(Folder.from(metadata), BwaAligner.NAMESPACE, bai(metadata.sampleName()))));
}
output = outputBuilder.build();
} else {
output = AlignmentOutput.builder().sample(metadata.sampleName()).status(PipelineStatus.FAILED).build();
}
trace.stop();
executorService.shutdown();
return output;
}
use of com.hartwig.pipeline.Arguments in project pipeline5 by hartwigmedical.
the class GoogleComputeEngineTest method stopsInstanceWithPersistentDisksUponFailure.
@Test
public void stopsInstanceWithPersistentDisksUponFailure() {
Arguments arguments = Arguments.testDefaultsBuilder().useLocalSsds(false).build();
victim = new GoogleComputeEngine(arguments, compute, z -> {
}, lifecycleManager, bucketWatcher, mock(Labels.class));
returnFailed();
victim.submit(runtimeBucket.getRuntimeBucket(), jobDefinition);
verify(lifecycleManager).stop(FIRST_ZONE_NAME, INSTANCE_NAME);
}
use of com.hartwig.pipeline.Arguments in project pipeline5 by hartwigmedical.
the class GoogleComputeEngineTest method disablesStartupScriptWhenInstanceWithPersistentDisksFailsRemotely.
@Test
public void disablesStartupScriptWhenInstanceWithPersistentDisksFailsRemotely() throws Exception {
Arguments arguments = Arguments.testDefaultsBuilder().useLocalSsds(false).build();
victim = new GoogleComputeEngine(arguments, compute, z -> {
}, lifecycleManager, bucketWatcher, mock(Labels.class));
returnFailed();
victim.submit(runtimeBucket.getRuntimeBucket(), jobDefinition);
verify(lifecycleManager).disableStartupScript(FIRST_ZONE_NAME, INSTANCE_NAME);
}
use of com.hartwig.pipeline.Arguments in project pipeline5 by hartwigmedical.
the class RuntimeBucketTest method appliesLabels.
@Test
public void appliesLabels() {
Arguments arguments = Arguments.testDefaultsBuilder().costCenterLabel("development").userLabel("username").build();
RuntimeBucket.from(storage, NAMESPACE, referenceRunMetadata(), arguments, Labels.of(arguments, TestInputs.defaultSomaticRunMetadata()));
assertThat(bucketInfo.getValue().getLabels()).isEqualTo(Map.of("cost_center", "development", "run_id", "test", "sample", "tumor", "user", "username"));
}
Aggregations