Examples with ResourceFiles com.hartwig.pipeline.resource.ResourceFiles used on opensource projects

Example 1 with ResourceFiles

use of com.hartwig.pipeline.resource.ResourceFiles in project pipeline5 by hartwigmedical.

the class LilacBatch method addSampleCommands.

private void addSampleCommands(final InputFileDescriptor runData, final BashStartupScript commands, final String runDirectory, final String sampleId, boolean hasRna) {
    final RemoteLocationsApi locationsApi = new RemoteLocationsApi(runData.billedProject(), sampleId);
    final LocalLocations localInput = new LocalLocations(new BamSliceDecorator(locationsApi));
    final String somaticVcf = localInput.getSomaticVariantsPurple();
    final String geneCopyNumber = localInput.getGeneCopyNumberTsv();
    final String tumorAlignment = localInput.getTumorAlignment();
    final String referenceAlignment = localInput.getReferenceAlignment();
    final String rnaAlignment = hasRna ? String.format("%s.rna.hla.bam", sampleId) : "";
    // download sample input files
    commands.addCommands(localInput.generateDownloadCommands());
    if (hasRna) {
        commands.addCommand(() -> format("gsutil -m cp gs://%s/%s/%s* %s", HLA_BAMS_BUCKET, sampleId, rnaAlignment, VmDirectories.INPUT));
    }
    // build Lilac arguments
    String sampleOutputDir = String.format("%s/%s/", VmDirectories.OUTPUT, sampleId);
    commands.addCommand(() -> format("mkdir -p %s", sampleOutputDir));
    final ResourceFiles resourceFiles = ResourceFilesFactory.buildResourceFiles(RefGenomeVersion.V37);
    StringJoiner lilacArgs = new StringJoiner(" ");
    lilacArgs.add(String.format(" -sample %s", sampleId));
    lilacArgs.add(String.format(" -resource_dir %s", VmDirectories.INPUT));
    lilacArgs.add(String.format(" -ref_genome %s", resourceFiles.refGenomeFile()));
    lilacArgs.add(String.format(" -reference_bam %s", referenceAlignment));
    lilacArgs.add(String.format(" -tumor_bam %s", tumorAlignment));
    if (hasRna) {
        lilacArgs.add(String.format(" -rna_bam %s/%s", VmDirectories.INPUT, rnaAlignment));
    }
    lilacArgs.add(String.format(" -output_dir %s", sampleOutputDir));
    lilacArgs.add(String.format(" -gene_copy_number_file %s", geneCopyNumber));
    lilacArgs.add(String.format(" -somatic_variants_file %s", somaticVcf));
    lilacArgs.add(String.format(" -threads %s", Bash.allCpus()));
    commands.addCommand(() -> format("java -Xmx%s -jar %s/%s %s", MAX_HEAP, VmDirectories.TOOLS, LILAC_JAR, lilacArgs.toString()));
    /*
        if(tumorOnly)
        {
            String tumorOutputDir = String.format("%s/%s/tumor", VmDirectories.OUTPUT, sampleId);
            commands.addCommand(() -> format("mkdir -p %s", tumorOutputDir));

            StringBuilder tumorLilacArgs = new StringBuilder();
            tumorLilacArgs.add(String.format(" -sample %s", sampleId));
            tumorLilacArgs.add(String.format(" -resource_dir %s", LOCAL_LILAC_RESOURCES));
            tumorLilacArgs.add(String.format(" -ref_genome %s", resourceFiles.refGenomeFile()));
            tumorLilacArgs.add(String.format(" -reference_bam %s", tumorAlignment));
            tumorLilacArgs.add(" -tumor_only");
            tumorLilacArgs.add(String.format(" -output_dir %s", tumorOutputDir));
            tumorLilacArgs.add(String.format(" -threads %s", Bash.allCpus()));

            commands.addCommand(() -> format("java -Xmx%s -jar %s/%s %s",
                    MAX_HEAP, VmDirectories.TOOLS, LILAC_JAR, tumorLilacArgs.toString()));
        }
        */
    String sampleRemoteOutputDir = String.format("gs://%s/%s/", LILAC_BATCH_BUCKET, runDirectory);
    commands.addCommand(() -> format("gsutil -m cp -r %s/%s/ %s", VmDirectories.OUTPUT, sampleId, sampleRemoteOutputDir));
}

Example 2 with ResourceFiles

use of com.hartwig.pipeline.resource.ResourceFiles in project pipeline5 by hartwigmedical.

the class PurpleRerun method bashCommands.

public List<BashCommand> bashCommands(RemoteLocations locations) {
    final LocalLocations batchInput = new LocalLocations(locations);
    final List<BashCommand> commands = Lists.newArrayList();
    final ResourceFiles resourceFiles = ResourceFilesFactory.buildResourceFiles(RefGenomeVersion.V37);
    final String tumorSampleName = batchInput.getTumor();
    final String referenceSampleName = batchInput.getReference();
    final String amberLocation = batchInput.getAmber();
    final String cobaltLocation = batchInput.getCobalt();
    final String sageSomaticLocation = batchInput.getSomaticVariantsSage();
    final String sageGermlineLocation = batchInput.getGermlineVariantsSage();
    final String gripssLocation = batchInput.getStructuralVariantsGripss();
    final String gripssRecoveryLocation = batchInput.getStructuralVariantsGripssRecovery();
    commands.addAll(batchInput.generateDownloadCommands());
    PurpleCommandBuilder builder = new PurpleCommandBuilder(resourceFiles, amberLocation, cobaltLocation, tumorSampleName, gripssLocation, gripssRecoveryLocation, sageSomaticLocation).addGermline(sageGermlineLocation);
    commands.add(builder.build());
    return commands;
}

Example 3 with ResourceFiles

use of com.hartwig.pipeline.resource.ResourceFiles in project pipeline5 by hartwigmedical.

the class SageBenchmarks method execute.

@Override
public VirtualMachineJobDefinition execute(final InputBundle inputs, final RuntimeBucket runtimeBucket, final BashStartupScript startupScript, final RuntimeFiles executionFlags) {
    InputFileDescriptor descriptor = inputs.get();
    final String[] inputData = descriptor.inputValue().split(",", -1);
    final String sampleId = inputData[0];
    final String referenceId = inputData[1];
    String runData = inputData[2];
    boolean runTumorNormal = runData.equals("TumorNormal");
    boolean runTumorOnly = runData.equals("TumorOnly");
    boolean runGermline = runData.equals("Germline");
    final ResourceFiles resourceFiles = ResourceFilesFactory.buildResourceFiles(RefGenomeVersion.V37);
    startupScript.addCommand(() -> format("gsutil -u hmf-crunch cp %s/%s/%s %s", BATCH_TOOLS_BUCKET, SAGE_DIR, SAGE_JAR, VmDirectories.TOOLS));
    /*
        startupScript.addCommand(() -> format("gsutil -u hmf-crunch cp %s/%s/%s %s",
                BATCH_TOOLS_BUCKET, PAVE_DIR, PAVE_JAR, VmDirectories.TOOLS));

        String ponFile = "SageGermlinePon.1000x.37.tsv.gz";

        startupScript.addCommand(() -> format("gsutil -u hmf-crunch cp %s/%s/%s %s",
                BATCH_RESOURCE_BUCKET, SAGE_DIR, ponFile, VmDirectories.INPUT));
        */
    // download tumor and ref BAMs as required
    String tumorBamFile = String.format("%s.bam", sampleId);
    String referenceBamFile = String.format("%s.bam", referenceId);
    if (runTumorNormal || runTumorOnly) {
        startupScript.addCommand(() -> format("gsutil -u hmf-crunch cp %s/%s/%s* %s", BATCH_BENCHMARKS_BUCKET, sampleId, tumorBamFile, VmDirectories.INPUT));
    }
    if (runTumorNormal || runGermline) {
        startupScript.addCommand(() -> format("gsutil -u hmf-crunch cp %s/%s/%s* %s", BATCH_BENCHMARKS_BUCKET, sampleId, referenceBamFile, VmDirectories.INPUT));
    }
    // run Sage
    final StringJoiner sageArgs = new StringJoiner(" ");
    if (runTumorNormal || runTumorOnly) {
        sageArgs.add(String.format("-tumor %s", sampleId));
        sageArgs.add(String.format("-tumor_bam %s/%s", VmDirectories.INPUT, tumorBamFile));
    } else if (runGermline) {
        sageArgs.add(String.format("-tumor %s", sampleId));
        sageArgs.add(String.format("-tumor_bam %s/%s", VmDirectories.INPUT, referenceBamFile));
    }
    if (runTumorNormal) {
        sageArgs.add(String.format("-reference %s", referenceId));
        sageArgs.add(String.format("-reference_bam %s/%s", VmDirectories.INPUT, referenceBamFile));
    } else if (runGermline) {
        sageArgs.add(String.format("-reference %s", referenceId));
        sageArgs.add(String.format("-reference_bam %s/%s", VmDirectories.INPUT, tumorBamFile));
    }
    if (runGermline) {
        sageArgs.add(String.format("-hotspots %s", resourceFiles.sageGermlineHotspots()));
        sageArgs.add(String.format("-panel_bed %s", resourceFiles.sageGermlineCodingPanel()));
    } else {
        sageArgs.add(String.format("-hotspots %s", resourceFiles.sageSomaticHotspots()));
        sageArgs.add(String.format("-panel_bed %s", resourceFiles.sageSomaticCodingPanel()));
    }
    sageArgs.add(String.format("-high_confidence_bed %s", resourceFiles.giabHighConfidenceBed()));
    sageArgs.add(String.format("-ref_genome %s", resourceFiles.refGenomeFile()));
    sageArgs.add(String.format("-ref_genome_version %s", resourceFiles.version().toString()));
    sageArgs.add(String.format("-ensembl_data_dir %s", resourceFiles.ensemblDataCache()));
    if (runGermline) {
        sageArgs.add("-panel_only");
        sageArgs.add("-hotspot_min_tumor_qual 50");
        sageArgs.add("-panel_min_tumor_qual 75");
        sageArgs.add("-hotspot_max_germline_vaf 100");
        sageArgs.add("-hotspot_max_germline_rel_raw_base_qual 100");
        sageArgs.add("-panel_max_germline_vaf 100");
        sageArgs.add("-panel_max_germline_rel_raw_base_qual 100");
        sageArgs.add("-mnv_filter_enabled false");
    }
    String sageVcf;
    if (runTumorOnly)
        sageVcf = String.format("%s/%s.sage.tumor_only.vcf.gz", VmDirectories.OUTPUT, sampleId);
    else if (runGermline)
        sageVcf = String.format("%s/%s.sage.germline.vcf.gz", VmDirectories.OUTPUT, sampleId);
    else
        sageVcf = String.format("%s/%s.sage.somatic.vcf.gz", VmDirectories.OUTPUT, sampleId);
    sageArgs.add(String.format("-out %s", sageVcf));
    sageArgs.add(String.format("-perf_warn_time 50"));
    sageArgs.add(String.format("-threads %s", Bash.allCpus()));
    startupScript.addCommand(() -> format("java -Xmx48G -jar %s/%s %s", VmDirectories.TOOLS, SAGE_JAR, sageArgs.toString()));
    /*
        // annotate with Pave - PON and gene impacts
        final StringJoiner paveArgs = new StringJoiner(" ");
        String ponFilters = "HOTSPOT:5:5;PANEL:2:5;UNKNOWN:2:0";

        final String paveVcf = String.format("%s/%s.sage.somatic.pon.pave.vcf.gz", VmDirectories.OUTPUT, sampleId);

        paveArgs.add(String.format("-sample %s", sampleId));
        paveArgs.add(String.format("-vcf_file %s", sageVcf)); // ponFilterVcf from BCF Tools

        paveArgs.add(String.format("-ref_genome %s", resourceFiles.refGenomeFile()));
        paveArgs.add(String.format("-ref_genome_version %s", resourceFiles.version().toString()));
        paveArgs.add(String.format("-driver_gene_panel %s", resourceFiles.driverGenePanel()));
        paveArgs.add(String.format("-ensembl_data_dir %s", resourceFiles.ensemblDataCache()));
        paveArgs.add(String.format("-pon_file %s/%s", VmDirectories.INPUT, ponFile));
        paveArgs.add(String.format("-pon_filters \"%s\"", ponFilters));
        paveArgs.add(String.format("-output_vcf_file %s", paveVcf));

        String paveJar = String.format("%s/%s", VmDirectories.TOOLS, PAVE_JAR);

        startupScript.addCommand(() -> format("java -jar %s %s", paveJar, paveArgs.toString()));
        */
    // upload output
    startupScript.addCommand(new OutputUpload(GoogleStorageLocation.of(runtimeBucket.name(), "sage"), executionFlags));
    return ImmutableVirtualMachineJobDefinition.builder().name("sage").startupCommand(startupScript).performanceProfile(custom(24, 64)).namespacedResults(ResultsDirectory.defaultDirectory()).build();
}

Example 4 with ResourceFiles

use of com.hartwig.pipeline.resource.ResourceFiles in project pipeline5 by hartwigmedical.

the class SageCompare method execute.

@Override
public VirtualMachineJobDefinition execute(final InputBundle inputs, final RuntimeBucket runtimeBucket, final BashStartupScript startupScript, final RuntimeFiles executionFlags) {
    InputFileDescriptor descriptor = inputs.get();
    final String[] sampleData = descriptor.inputValue().split(",", -1);
    final String sampleId = sampleData[0];
    String runTypes = sampleData.length > 1 ? sampleData[1] : RUN_BOTH;
    boolean runBoth = runTypes.equalsIgnoreCase(RUN_BOTH);
    boolean cramVsBam = runTypes.equalsIgnoreCase(RUN_CRAM_VS_BAM);
    boolean runOld = runBoth || runTypes.equalsIgnoreCase(RUN_OLD);
    boolean runNew = runBoth || cramVsBam || runTypes.equalsIgnoreCase(RUN_NEW);
    final ResourceFiles resourceFiles = ResourceFilesFactory.buildResourceFiles(RefGenomeVersion.V37);
    startupScript.addCommand(() -> format("gsutil -u hmf-crunch cp %s/%s/%s %s", BATCH_TOOLS_BUCKET, SAGE_DIR, SAGE_JAR, VmDirectories.TOOLS));
    final RemoteLocationsApi locations = new RemoteLocationsApi("hmf-crunch", sampleId);
    String[] tumorCramData = getCramFileData(locations.getTumorAlignment());
    String tumorCramFile = tumorCramData[CRAM_FILENAME];
    startupScript.addCommand(() -> format("gsutil -u hmf-crunch cp gs://%s* %s", tumorCramData[CRAM_FULL_PATH], VmDirectories.INPUT));
    String referenceId = locations.getReference();
    String[] refCramData = getCramFileData(locations.getReferenceAlignment());
    String refCramFile = refCramData[CRAM_FILENAME];
    startupScript.addCommand(() -> format("gsutil -u hmf-crunch cp gs://%s* %s", refCramData[CRAM_FULL_PATH], VmDirectories.INPUT));
    // download tumor CRAM
    String localTumorCram = String.format("%s/%s", VmDirectories.INPUT, tumorCramFile);
    String localRefCram = String.format("%s/%s", VmDirectories.INPUT, refCramFile);
    // and convert to BAM
    startupScript.addCommands(cramToBam(localTumorCram));
    startupScript.addCommands(cramToBam(localRefCram));
    String localTumorBam = localTumorCram.replace("cram", "bam");
    String localRefBam = localRefCram.replace("cram", "bam");
    if (runOld) {
        final String oldSageVcf = String.format("%s/%s.sage.somatic.vcf.gz", VmDirectories.OUTPUT, sampleId);
        // run old Sage
        final StringJoiner oldSageArgs = new StringJoiner(" ");
        oldSageArgs.add(String.format("-tumor %s", sampleId));
        oldSageArgs.add(String.format("-tumor_bam %s", localTumorBam));
        oldSageArgs.add(String.format("-reference %s", referenceId));
        oldSageArgs.add(String.format("-reference_bam %s", localRefBam));
        oldSageArgs.add(String.format("-hotspots %s", resourceFiles.sageSomaticHotspots()));
        oldSageArgs.add(String.format("-panel_bed %s", resourceFiles.sageSomaticCodingPanel()));
        oldSageArgs.add(String.format("-high_confidence_bed %s", resourceFiles.giabHighConfidenceBed()));
        oldSageArgs.add(String.format("-ref_genome %s", resourceFiles.refGenomeFile()));
        oldSageArgs.add("-assembly hg19");
        oldSageArgs.add("-bqr_plot false");
        oldSageArgs.add(String.format("-out %s", oldSageVcf));
        oldSageArgs.add(String.format("-threads %s", Bash.allCpus()));
        // oldSageArgs.add("-chr 14");
        String oldSageJar = String.format("sage/%s/sage.jar", Versions.SAGE);
        startupScript.addCommand(() -> format("java -Xmx48G -jar %s/%s %s", VmDirectories.TOOLS, oldSageJar, oldSageArgs.toString()));
    }
    if (runNew) {
        final String newSageVcf = String.format("%s/%s.sage.somatic.vcf.gz", VmDirectories.OUTPUT, sampleId);
        final StringJoiner newSageArgs = new StringJoiner(" ");
        newSageArgs.add(String.format("-tumor %s", sampleId));
        newSageArgs.add(String.format("-tumor_bam %s", localTumorBam));
        newSageArgs.add(String.format("-reference %s", referenceId));
        newSageArgs.add(String.format("-reference_bam %s", localRefBam));
        newSageArgs.add(String.format("-hotspots %s", resourceFiles.sageSomaticHotspots()));
        newSageArgs.add(String.format("-panel_bed %s", resourceFiles.sageSomaticCodingPanel()));
        newSageArgs.add(String.format("-high_confidence_bed %s", resourceFiles.giabHighConfidenceBed()));
        newSageArgs.add(String.format("-ref_genome %s", resourceFiles.refGenomeFile()));
        newSageArgs.add(String.format("-ref_genome_version %s", resourceFiles.version().toString()));
        newSageArgs.add(String.format("-ensembl_data_dir %s", resourceFiles.ensemblDataCache()));
        newSageArgs.add(String.format("-perf_warn_time 50"));
        newSageArgs.add(String.format("-log_debug"));
        newSageArgs.add(String.format("-out %s", newSageVcf));
        newSageArgs.add(String.format("-threads %s", Bash.allCpus()));
        startupScript.addCommand(() -> format("java -Xmx48G -jar %s/%s %s", VmDirectories.TOOLS, SAGE_JAR, newSageArgs.toString()));
    }
    if (cramVsBam) {
        final String newCramSageVcf = String.format("%s/%s.sage.somatic.cram.vcf.gz", VmDirectories.OUTPUT, sampleId);
        final StringJoiner newSageArgs = new StringJoiner(" ");
        newSageArgs.add(String.format("-tumor %s", sampleId));
        newSageArgs.add(String.format("-tumor_bam %s", localTumorCram));
        newSageArgs.add(String.format("-reference %s", referenceId));
        newSageArgs.add(String.format("-reference_bam %s", localRefCram));
        newSageArgs.add(String.format("-hotspots %s", resourceFiles.sageSomaticHotspots()));
        newSageArgs.add(String.format("-panel_bed %s", resourceFiles.sageSomaticCodingPanel()));
        newSageArgs.add(String.format("-high_confidence_bed %s", resourceFiles.giabHighConfidenceBed()));
        newSageArgs.add(String.format("-ref_genome %s", resourceFiles.refGenomeFile()));
        newSageArgs.add(String.format("-ref_genome_version %s", resourceFiles.version().toString()));
        newSageArgs.add(String.format("-ensembl_data_dir %s", resourceFiles.ensemblDataCache()));
        newSageArgs.add(String.format("-perf_warn_time 50"));
        newSageArgs.add(String.format("-log_debug"));
        newSageArgs.add(String.format("-out %s", newCramSageVcf));
        newSageArgs.add(String.format("-threads %s", Bash.allCpus()));
        startupScript.addCommand(() -> format("java -Xmx48G -jar %s/%s %s", VmDirectories.TOOLS, SAGE_JAR, newSageArgs.toString()));
    }
    // upload output
    startupScript.addCommand(new OutputUpload(GoogleStorageLocation.of(runtimeBucket.name(), "sage"), executionFlags));
    return ImmutableVirtualMachineJobDefinition.builder().name("sage").startupCommand(startupScript).performanceProfile(custom(24, 64)).namespacedResults(ResultsDirectory.defaultDirectory()).build();
}

Example 5 with ResourceFiles

use of com.hartwig.pipeline.resource.ResourceFiles in project pipeline5 by hartwigmedical.

the class GridssRerun method execute.

@Override
public VirtualMachineJobDefinition execute(final InputBundle inputs, final RuntimeBucket runtimeBucket, final BashStartupScript commands, final RuntimeFiles executionFlags) {
    // Inputs
    final ResourceFiles resourceFiles = ResourceFilesFactory.buildResourceFiles(RefGenomeVersion.V37);
    final String set = inputs.get("set").inputValue();
    final String tumorSampleName = inputs.get("tumor_sample").inputValue();
    final String referenceSampleName = inputs.get("reference_sample").inputValue();
    final InputFileDescriptor remoteTumorFile = inputs.get("tumor_cram");
    final InputFileDescriptor remoteReferenceFile = inputs.get("ref_cram");
    final InputFileDescriptor runData = inputs.get();
    final RemoteLocationsApi locationsApi = new RemoteLocationsApi(runData.billedProject(), tumorSampleName);
    InputDownload tumorBamDownload = new InputDownload(locationsApi.getTumorAlignment());
    InputDownload tumorBamIndexDownload = new InputDownload(locationsApi.getTumorAlignmentIndex());
    InputDownload referenceBamDownload = new InputDownload(locationsApi.getReferenceAlignment());
    InputDownload referenceBamIndexDownload = new InputDownload(locationsApi.getReferenceAlignmentIndex());
    final InputFileDescriptor remoteTumorIndex = remoteTumorFile.index();
    final InputFileDescriptor remoteReferenceIndex = remoteReferenceFile.index();
    final String localTumorFile = localFilename(remoteTumorFile);
    final String localReferenceFile = localFilename(remoteReferenceFile);
    final String tumorBamPath = localTumorFile.replace("cram", "bam");
    final String refBamPath = localReferenceFile.replace("cram", "bam");
    Driver driver = new Driver(resourceFiles, VmDirectories.outputFile(tumorSampleName + ".assembly.bam")).tumorSample(tumorSampleName, tumorBamPath).referenceSample(referenceSampleName, refBamPath);
    GridssAnnotation viralAnnotation = new GridssAnnotation(resourceFiles, false);
    SubStageInputOutput unfilteredVcfOutput = driver.andThen(viralAnnotation).apply(SubStageInputOutput.empty(tumorSampleName));
    final OutputFile unfilteredVcf = unfilteredVcfOutput.outputFile();
    final OutputFile unfilteredVcfIndex = unfilteredVcf.index(".tbi");
    final GoogleStorageLocation unfilteredVcfRemoteLocation = remoteUnfilteredVcfArchivePath(set, tumorSampleName);
    final GoogleStorageLocation unfilteredVcfIndexRemoteLocation = index(unfilteredVcfRemoteLocation, ".tbi");
    // COMMANDS
    commands.addCommand(new ExportPathCommand(new BwaCommand()));
    commands.addCommand(new ExportPathCommand(new SamtoolsCommand()));
    commands.addCommand(() -> remoteTumorFile.toCommandForm(localTumorFile));
    commands.addCommand(() -> remoteTumorIndex.toCommandForm(localFilename(remoteTumorIndex)));
    commands.addCommand(() -> remoteReferenceFile.toCommandForm(localReferenceFile));
    commands.addCommand(() -> remoteReferenceIndex.toCommandForm(localFilename(remoteReferenceIndex)));
    if (!localTumorFile.equals(tumorBamPath)) {
        commands.addCommands(cramToBam(localTumorFile));
    }
    if (!localReferenceFile.equals(refBamPath)) {
        commands.addCommands(cramToBam(localReferenceFile));
    }
    commands.addCommands(unfilteredVcfOutput.bash());
    commands.addCommand(() -> unfilteredVcf.copyToRemoteLocation(unfilteredVcfRemoteLocation));
    commands.addCommand(() -> unfilteredVcfIndex.copyToRemoteLocation(unfilteredVcfIndexRemoteLocation));
    commands.addCommand(new OutputUpload(GoogleStorageLocation.of(runtimeBucket.name(), "gridss"), executionFlags));
    return VirtualMachineJobDefinition.structuralCalling(commands, ResultsDirectory.defaultDirectory());
}