Examples with RuntimeBucket com.hartwig.pipeline.storage.RuntimeBucket used on opensource projects

Example 6 with RuntimeBucket

use of com.hartwig.pipeline.storage.RuntimeBucket in project pipeline5 by hartwigmedical.

the class ZippedVcfAndIndexComponentTest method copiesVcfAndTbiToOutpput.

@Test
public void copiesVcfAndTbiToOutpput() {
    Storage storage = mock(Storage.class);
    RuntimeBucket runtimeBucket = MockRuntimeBucket.test().getRuntimeBucket();
    Bucket reportBucket = mock(Bucket.class);
    when(reportBucket.getName()).thenReturn(OUTPUT_BUCKET);
    ArgumentCaptor<String> sourceBlobCaptor = ArgumentCaptor.forClass(String.class);
    ArgumentCaptor<String> targetBucketCaptor = ArgumentCaptor.forClass(String.class);
    ArgumentCaptor<String> targetBlobCaptor = ArgumentCaptor.forClass(String.class);
    ZippedVcfAndIndexComponent victim = new ZippedVcfAndIndexComponent(runtimeBucket, "test", Folder.from(TestInputs.referenceRunMetadata()), TEST_VCF, TEST_VCF, ResultsDirectory.defaultDirectory());
    victim.addToReport(storage, reportBucket, "test_set");
    verify(runtimeBucket, times(2)).copyOutOf(sourceBlobCaptor.capture(), targetBucketCaptor.capture(), targetBlobCaptor.capture());
    assertThat(sourceBlobCaptor.getAllValues().get(0)).isEqualTo("results/test.vcf");
    assertThat(sourceBlobCaptor.getAllValues().get(1)).isEqualTo("results/test.vcf.tbi");
    assertThat(targetBucketCaptor.getValue()).isEqualTo(OUTPUT_BUCKET);
    assertThat(targetBlobCaptor.getAllValues().get(0)).isEqualTo("test_set/reference/test/test.vcf");
    assertThat(targetBlobCaptor.getAllValues().get(1)).isEqualTo("test_set/reference/test/test.vcf.tbi");
}

Example 7 with RuntimeBucket

use of com.hartwig.pipeline.storage.RuntimeBucket in project pipeline5 by hartwigmedical.

the class QuotaConstrainedComputeEngine method submit.

@Override
public PipelineStatus submit(final RuntimeBucket bucket, final VirtualMachineJobDefinition jobDefinition, final String discriminator) {
    VirtualMachineJobDefinition constrained = jobDefinition;
    MachineType machineType = jobDefinition.performanceProfile().machineType();
    try {
        QuotaBucket regionalQuota = serviceUsage.services().consumerQuotaMetrics().limits().get(cpuQuotaName(project)).execute().getQuotaBuckets().stream().filter(b -> b.getDimensions() != null).filter(b -> region(b) != null).filter(b -> region(b).equals(region)).findFirst().orElseThrow();
        int maxCPU = (int) (regionalQuota.getEffectiveLimit().intValue() * constrainByPercentage);
        if (machineType.cpus() > maxCPU) {
            double reductionRatio = (double) maxCPU / machineType.cpus();
            constrained = VirtualMachineJobDefinition.builder().from(constrained).performanceProfile(VirtualMachinePerformanceProfile.custom(maxCPU, (int) (machineType.memoryGB() * reductionRatio))).build();
        }
        return decorated.submit(bucket, constrained, discriminator);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 8 with RuntimeBucket

use of com.hartwig.pipeline.storage.RuntimeBucket in project pipeline5 by hartwigmedical.

the class StageTest method setUp.

@Before
public void setUp() throws Exception {
    persistedDataset = new TestPersistedDataset();
    victim = createVictim();
    String runtimeBucketName = expectedRuntimeBucketName() + "/" + victim.namespace();
    storage = mock(Storage.class);
    bucket = mock(Bucket.class);
    when(storage.get(runtimeBucketName)).thenReturn(bucket);
    CopyWriter copyWriter = mock(CopyWriter.class);
    when(storage.copy(any())).thenReturn(copyWriter);
    storage = mock(Storage.class);
    runtimeBucket = mock(RuntimeBucket.class);
    when(runtimeBucket.name()).thenReturn(runtimeBucketName);
    when(runtimeBucket.get(any())).thenReturn(mock(Blob.class));
}

Example 9 with RuntimeBucket

use of com.hartwig.pipeline.storage.RuntimeBucket in project pipeline5 by hartwigmedical.

the class RunLogComponentTest method copiesRunLogIntoReportBucket.

@Test
public void copiesRunLogIntoReportBucket() {
    Storage storage = mock(Storage.class);
    RuntimeBucket runtimeBucket = MockRuntimeBucket.test().getRuntimeBucket();
    Bucket reportBucket = mock(Bucket.class);
    when(reportBucket.getName()).thenReturn(REPORT_BUCKET);
    ArgumentCaptor<String> sourceBlobCaptor = ArgumentCaptor.forClass(String.class);
    ArgumentCaptor<String> targetBucketCaptor = ArgumentCaptor.forClass(String.class);
    ArgumentCaptor<String> targetBlobCaptor = ArgumentCaptor.forClass(String.class);
    RunLogComponent victim = new RunLogComponent(runtimeBucket, "test", Folder.from(TestInputs.referenceRunMetadata()), ResultsDirectory.defaultDirectory());
    victim.addToReport(storage, reportBucket, "test_set");
    verify(runtimeBucket, times(1)).copyOutOf(sourceBlobCaptor.capture(), targetBucketCaptor.capture(), targetBlobCaptor.capture());
    assertThat(sourceBlobCaptor.getValue()).isEqualTo("results/run.log");
    assertThat(targetBucketCaptor.getValue()).isEqualTo(REPORT_BUCKET);
    assertThat(targetBlobCaptor.getValue()).isEqualTo("test_set/reference/test/run.log");
}