Examples with StorageCow bio.terra.cloudres.google.storage.StorageCow used on opensource projects

Example 1 with StorageCow

use of bio.terra.cloudres.google.storage.StorageCow in project terra-workspace-manager by DataBiosphere.

the class ControlledResourceServiceTest method createGcsBucketDo.

@Test
@DisabledIfEnvironmentVariable(named = "TEST_ENV", matches = BUFFER_SERVICE_DISABLED_ENVS_REG_EX)
void createGcsBucketDo() throws Exception {
    ControlledGcsBucketResource resource = ControlledResourceFixtures.makeDefaultControlledGcsBucketBuilder(workspace.getWorkspaceId()).build();
    // Test idempotency of bucket-specific steps by retrying them once.
    Map<String, StepStatus> retrySteps = new HashMap<>();
    retrySteps.put(CreateGcsBucketStep.class.getName(), StepStatus.STEP_RESULT_FAILURE_RETRY);
    retrySteps.put(GcsBucketCloudSyncStep.class.getName(), StepStatus.STEP_RESULT_FAILURE_RETRY);
    jobService.setFlightDebugInfoForTest(FlightDebugInfo.newBuilder().doStepFailures(retrySteps).build());
    ControlledGcsBucketResource createdBucket = controlledResourceService.createControlledResourceSync(resource, null, user.getAuthenticatedRequest(), ControlledResourceFixtures.getGoogleBucketCreationParameters()).castByEnum(WsmResourceType.CONTROLLED_GCP_GCS_BUCKET);
    assertEquals(resource, createdBucket);
    StorageCow storageCow = crlService.createStorageCow(projectId);
    BucketInfo cloudBucket = storageCow.get(resource.getBucketName()).getBucketInfo();
    assertEquals(DEFAULT_REGION, cloudBucket.getLocation().toLowerCase());
    assertEquals(resource, controlledResourceService.getControlledResource(workspace.getWorkspaceId(), resource.getResourceId(), user.getAuthenticatedRequest()));
}

Example 2 with StorageCow

use of bio.terra.cloudres.google.storage.StorageCow in project terra-workspace-manager by DataBiosphere.

the class ControlledResourceServiceTest method deleteGcsBucketDo.

@Test
@DisabledIfEnvironmentVariable(named = "TEST_ENV", matches = BUFFER_SERVICE_DISABLED_ENVS_REG_EX)
void deleteGcsBucketDo() throws Exception {
    ControlledGcsBucketResource createdBucket = createDefaultSharedGcsBucket(workspace, user);
    // Test idempotency of bucket-specific delete step by retrying it once.
    Map<String, StepStatus> retrySteps = new HashMap<>();
    retrySteps.put(DeleteGcsBucketStep.class.getName(), StepStatus.STEP_RESULT_FAILURE_RETRY);
    jobService.setFlightDebugInfoForTest(FlightDebugInfo.newBuilder().doStepFailures(retrySteps).build());
    String jobId = controlledResourceService.deleteControlledResourceAsync(new ApiJobControl().id(UUID.randomUUID().toString()), workspace.getWorkspaceId(), createdBucket.getResourceId(), "fake result path", user.getAuthenticatedRequest());
    jobService.waitForJob(jobId);
    assertEquals(FlightStatus.SUCCESS, stairwayComponent.get().getFlightState(jobId).getFlightStatus());
    // Validate the bucket does not exist.
    StorageCow storageCow = crlService.createStorageCow(projectId);
    assertNull(storageCow.get(createdBucket.getBucketName()));
    assertThrows(ResourceNotFoundException.class, () -> controlledResourceService.getControlledResource(workspace.getWorkspaceId(), createdBucket.getResourceId(), user.getAuthenticatedRequest()));
}

Example 3 with StorageCow

use of bio.terra.cloudres.google.storage.StorageCow in project terra-workspace-manager by DataBiosphere.

the class ControlledResourceServiceTest method createGcsBucketUndo.

@Test
@DisabledIfEnvironmentVariable(named = "TEST_ENV", matches = BUFFER_SERVICE_DISABLED_ENVS_REG_EX)
void createGcsBucketUndo() throws Exception {
    ControlledGcsBucketResource resource = ControlledResourceFixtures.makeDefaultControlledGcsBucketBuilder(workspace.getWorkspaceId()).build();
    // Test idempotency of bucket-specific undo steps by retrying them once. Fail at the end of
    // the flight to ensure undo steps work properly.
    Map<String, StepStatus> retrySteps = new HashMap<>();
    retrySteps.put(CreateGcsBucketStep.class.getName(), StepStatus.STEP_RESULT_FAILURE_RETRY);
    retrySteps.put(GcsBucketCloudSyncStep.class.getName(), StepStatus.STEP_RESULT_FAILURE_RETRY);
    jobService.setFlightDebugInfoForTest(FlightDebugInfo.newBuilder().undoStepFailures(retrySteps).lastStepFailure(true).build());
    // Service methods which wait for a flight to complete will throw an
    // InvalidResultStateException when that flight fails without a cause, which occurs when a
    // flight fails via debugInfo.
    assertThrows(InvalidResultStateException.class, () -> controlledResourceService.createControlledResourceSync(resource, null, user.getAuthenticatedRequest(), ControlledResourceFixtures.getGoogleBucketCreationParameters()));
    // Validate the bucket does not exist.
    StorageCow storageCow = crlService.createStorageCow(projectId);
    assertNull(storageCow.get(resource.getBucketName()));
    assertThrows(ResourceNotFoundException.class, () -> controlledResourceService.getControlledResource(workspace.getWorkspaceId(), resource.getResourceId(), user.getAuthenticatedRequest()));
}

Example 4 with StorageCow

use of bio.terra.cloudres.google.storage.StorageCow in project terra-workspace-manager by DataBiosphere.

the class CreateGcsBucketStep method doStep.

@Override
public StepResult doStep(FlightContext flightContext) throws InterruptedException, RetryException {
    FlightMap inputMap = flightContext.getInputParameters();
    ApiGcpGcsBucketCreationParameters creationParameters = inputMap.get(CREATION_PARAMETERS, ApiGcpGcsBucketCreationParameters.class);
    String projectId = gcpCloudContextService.getRequiredGcpProject(resource.getWorkspaceId());
    BucketInfo.Builder bucketInfoBuilder = BucketInfo.newBuilder(resource.getBucketName()).setLocation(Optional.ofNullable(creationParameters.getLocation()).orElse(DEFAULT_REGION));
    // Remaining creation parameters are optional
    Optional.ofNullable(creationParameters.getDefaultStorageClass()).map(GcsApiConversions::toGcsApi).ifPresent(bucketInfoBuilder::setStorageClass);
    bucketInfoBuilder.setLifecycleRules(Optional.ofNullable(creationParameters.getLifecycle()).map(GcsApiConversions::toGcsApiRulesList).orElse(Collections.emptyList()));
    BucketInfo.IamConfiguration iamConfiguration = BucketInfo.IamConfiguration.newBuilder().setIsUniformBucketLevelAccessEnabled(true).build();
    bucketInfoBuilder.setIamConfiguration(iamConfiguration);
    // Uniqueness within the project is already verified in WSM's DB earlier in this flight.
    try {
        Optional<Bucket> existingBucket = getBucket(resource.getBucketName());
        if (existingBucket.isEmpty()) {
            StorageCow storageCow = crlService.createStorageCow(projectId);
            storageCow.create(bucketInfoBuilder.build());
        } else if (bucketInProject(existingBucket.get(), projectId)) {
            logger.info("Bucket {} already exists in workspace project, this is a Stairway retry. Continuing.", resource.getBucketName());
        } else {
            throw new DuplicateResourceException("The provided bucket name is already in use, please choose another.");
        }
    } catch (StorageException storageException) {
        // in GCP's global bucket namespace, even if we don't have permission to GET it.
        if (storageException.getCode() == HttpStatus.SC_CONFLICT) {
            throw new DuplicateResourceException("The provided bucket name is already in use, please choose another.", storageException);
        }
        if (storageException.getCode() == HttpStatus.SC_BAD_REQUEST) {
            throw new BadRequestException("Received 400 BAD_REQUEST exception when creating a new gcs-bucket", storageException);
        }
        // Other cloud errors are unexpected here, rethrow.
        throw storageException;
    }
    return StepResult.getStepResultSuccess();
}

Example 5 with StorageCow

use of bio.terra.cloudres.google.storage.StorageCow in project terra-workspace-manager by DataBiosphere.

the class CreateGcsBucketStep method undoStep.

@Override
public StepResult undoStep(FlightContext flightContext) throws InterruptedException {
    String projectId = gcpCloudContextService.getRequiredGcpProject(resource.getWorkspaceId());
    // WSM should only attempt to delete the buckets it created, so it does nothing if the bucket
    // exists outside the current project. We can guarantee another flight did not create this
    // bucket because uniqueness within the project is already verified in WSM's DB earlier in this
    // flight.
    Optional<Bucket> existingBucket = getBucket(resource.getBucketName());
    if (existingBucket.isPresent() && bucketInProject(existingBucket.get(), projectId)) {
        final StorageCow storageCow = crlService.createStorageCow(projectId);
        storageCow.delete(resource.getBucketName());
    }
    return StepResult.getStepResultSuccess();
}