use of bio.terra.cloudres.google.bigquery.BigQueryCow in project terra-workspace-manager by DataBiosphere.
the class ControlledResourceServiceTest method createBqDatasetDo.
@Test
@DisabledIfEnvironmentVariable(named = "TEST_ENV", matches = BUFFER_SERVICE_DISABLED_ENVS_REG_EX)
void createBqDatasetDo() throws Exception {
String datasetId = ControlledResourceFixtures.uniqueDatasetId();
String location = "us-central1";
Integer defaultTableLifetimeSec = 5900;
Integer defaultPartitionLifetimeSec = 5901;
ApiGcpBigQueryDatasetCreationParameters creationParameters = new ApiGcpBigQueryDatasetCreationParameters().datasetId(datasetId).location(location).defaultTableLifetime(defaultTableLifetimeSec).defaultPartitionLifetime(defaultPartitionLifetimeSec);
ControlledBigQueryDatasetResource resource = ControlledResourceFixtures.makeDefaultControlledBigQueryBuilder(workspace.getWorkspaceId()).datasetName(datasetId).build();
// Test idempotency of dataset-specific step by retrying it once.
Map<String, StepStatus> retrySteps = new HashMap<>();
retrySteps.put(CreateBigQueryDatasetStep.class.getName(), StepStatus.STEP_RESULT_FAILURE_RETRY);
jobService.setFlightDebugInfoForTest(FlightDebugInfo.newBuilder().doStepFailures(retrySteps).build());
ControlledBigQueryDatasetResource createdDataset = controlledResourceService.createControlledResourceSync(resource, null, user.getAuthenticatedRequest(), creationParameters).castByEnum(WsmResourceType.CONTROLLED_GCP_BIG_QUERY_DATASET);
assertEquals(resource, createdDataset);
BigQueryCow bqCow = crlService.createWsmSaBigQueryCow();
Dataset cloudDataset = bqCow.datasets().get(projectId, createdDataset.getDatasetName()).execute();
assertEquals(location, cloudDataset.getLocation());
assertEquals(defaultTableLifetimeSec * 1000L, cloudDataset.getDefaultTableExpirationMs());
assertEquals(defaultPartitionLifetimeSec * 1000L, cloudDataset.getDefaultPartitionExpirationMs());
assertEquals(resource, controlledResourceService.getControlledResource(workspace.getWorkspaceId(), resource.getResourceId(), user.getAuthenticatedRequest()));
}
use of bio.terra.cloudres.google.bigquery.BigQueryCow in project terra-workspace-manager by DataBiosphere.
the class ControlledResourceServiceTest method validateBigQueryDatasetCloudMetadata.
/**
* Lookup the location and expiration times stored on the cloud for a BigQuery dataset, and assert
* they match the given values.
*/
private void validateBigQueryDatasetCloudMetadata(String projectId, String datasetId, String location, Integer defaultTableExpirationSec, Integer defaultPartitionExpirationSec) throws IOException {
BigQueryCow bqCow = crlService.createWsmSaBigQueryCow();
Dataset cloudDataset = bqCow.datasets().get(projectId, datasetId).execute();
assertEquals(location, cloudDataset.getLocation());
if (defaultTableExpirationSec == null) {
assertNull(cloudDataset.getDefaultTableExpirationMs());
} else {
assertEquals(defaultTableExpirationSec * 1000, cloudDataset.getDefaultTableExpirationMs());
}
if (defaultPartitionExpirationSec == null) {
assertNull(cloudDataset.getDefaultPartitionExpirationMs());
} else {
assertEquals(defaultPartitionExpirationSec * 1000, cloudDataset.getDefaultPartitionExpirationMs());
}
}
use of bio.terra.cloudres.google.bigquery.BigQueryCow in project terra-workspace-manager by DataBiosphere.
the class ControlledResourceServiceTest method deleteBqDatasetUndo.
@Test
@DisabledIfEnvironmentVariable(named = "TEST_ENV", matches = BUFFER_SERVICE_DISABLED_ENVS_REG_EX)
void deleteBqDatasetUndo() throws Exception {
String datasetId = ControlledResourceFixtures.uniqueDatasetId();
String location = "us-central1";
ApiGcpBigQueryDatasetCreationParameters creationParameters = new ApiGcpBigQueryDatasetCreationParameters().datasetId(datasetId).location(location);
ControlledBigQueryDatasetResource resource = ControlledResourceFixtures.makeDefaultControlledBigQueryBuilder(workspace.getWorkspaceId()).datasetName(datasetId).build();
ControlledBigQueryDatasetResource createdDataset = controlledResourceService.createControlledResourceSync(resource, null, user.getAuthenticatedRequest(), creationParameters).castByEnum(WsmResourceType.CONTROLLED_GCP_BIG_QUERY_DATASET);
assertEquals(resource, createdDataset);
// None of the steps on this flight are undoable, so even with lastStepFailure set to true we
// should expect the resource to really be deleted.
jobService.setFlightDebugInfoForTest(FlightDebugInfo.newBuilder().lastStepFailure(true).build());
assertThrows(InvalidResultStateException.class, () -> controlledResourceService.deleteControlledResourceSync(resource.getWorkspaceId(), resource.getResourceId(), user.getAuthenticatedRequest()));
BigQueryCow bqCow = crlService.createWsmSaBigQueryCow();
GoogleJsonResponseException getException = assertThrows(GoogleJsonResponseException.class, () -> bqCow.datasets().get(projectId, resource.getDatasetName()).execute());
assertEquals(HttpStatus.NOT_FOUND.value(), getException.getStatusCode());
assertThrows(ResourceNotFoundException.class, () -> controlledResourceService.getControlledResource(workspace.getWorkspaceId(), resource.getResourceId(), user.getAuthenticatedRequest()));
}
use of bio.terra.cloudres.google.bigquery.BigQueryCow in project terra-workspace-manager by DataBiosphere.
the class DeleteBigQueryDatasetStep method doStep.
@Override
public StepResult doStep(FlightContext flightContext) throws InterruptedException, RetryException {
final GcpCloudContext gcpCloudContext = flightContext.getWorkingMap().get(ControlledResourceKeys.GCP_CLOUD_CONTEXT, GcpCloudContext.class);
String projectId = gcpCloudContext.getGcpProjectId();
BigQueryCow bqCow = crlService.createWsmSaBigQueryCow();
try {
// With deleteContents set to true, this will delete the dataset even if it still has tables.
bqCow.datasets().delete(projectId, resource.getDatasetName()).setDeleteContents(true).execute();
} catch (GoogleJsonResponseException e) {
if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
logger.info("BQ dataset {} in project {} already deleted", resource.getDatasetName(), projectId);
return StepResult.getStepResultSuccess();
}
return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
} catch (IOException e) {
return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
}
return StepResult.getStepResultSuccess();
}
use of bio.terra.cloudres.google.bigquery.BigQueryCow in project terra-workspace-manager by DataBiosphere.
the class CreateBigQueryDatasetStep method undoStep.
@Override
public StepResult undoStep(FlightContext flightContext) throws InterruptedException {
String projectId = gcpCloudContextService.getRequiredGcpProject(resource.getWorkspaceId());
BigQueryCow bqCow = crlService.createWsmSaBigQueryCow();
try {
// With deleteContents set to true, this will delete the dataset even if other steps fail
// to clean up tables or data.
bqCow.datasets().delete(projectId, resource.getDatasetName()).setDeleteContents(true).execute();
} catch (GoogleJsonResponseException e) {
// Stairway steps may run multiple times, so we may already have deleted this resource.
if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
logger.info("BQ dataset {} in project {} already deleted", resource.getDatasetName(), projectId);
return StepResult.getStepResultSuccess();
}
return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
} catch (IOException e) {
return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
}
return StepResult.getStepResultSuccess();
}
Aggregations