Search in sources :

Example 1 with BigQueryCow

use of bio.terra.cloudres.google.bigquery.BigQueryCow in project terra-workspace-manager by DataBiosphere.

the class ControlledResourceServiceTest method createBqDatasetDo.

@Test
@DisabledIfEnvironmentVariable(named = "TEST_ENV", matches = BUFFER_SERVICE_DISABLED_ENVS_REG_EX)
void createBqDatasetDo() throws Exception {
    String datasetId = ControlledResourceFixtures.uniqueDatasetId();
    String location = "us-central1";
    Integer defaultTableLifetimeSec = 5900;
    Integer defaultPartitionLifetimeSec = 5901;
    ApiGcpBigQueryDatasetCreationParameters creationParameters = new ApiGcpBigQueryDatasetCreationParameters().datasetId(datasetId).location(location).defaultTableLifetime(defaultTableLifetimeSec).defaultPartitionLifetime(defaultPartitionLifetimeSec);
    ControlledBigQueryDatasetResource resource = ControlledResourceFixtures.makeDefaultControlledBigQueryBuilder(workspace.getWorkspaceId()).datasetName(datasetId).build();
    // Test idempotency of dataset-specific step by retrying it once.
    Map<String, StepStatus> retrySteps = new HashMap<>();
    retrySteps.put(CreateBigQueryDatasetStep.class.getName(), StepStatus.STEP_RESULT_FAILURE_RETRY);
    jobService.setFlightDebugInfoForTest(FlightDebugInfo.newBuilder().doStepFailures(retrySteps).build());
    ControlledBigQueryDatasetResource createdDataset = controlledResourceService.createControlledResourceSync(resource, null, user.getAuthenticatedRequest(), creationParameters).castByEnum(WsmResourceType.CONTROLLED_GCP_BIG_QUERY_DATASET);
    assertEquals(resource, createdDataset);
    BigQueryCow bqCow = crlService.createWsmSaBigQueryCow();
    Dataset cloudDataset = bqCow.datasets().get(projectId, createdDataset.getDatasetName()).execute();
    assertEquals(location, cloudDataset.getLocation());
    assertEquals(defaultTableLifetimeSec * 1000L, cloudDataset.getDefaultTableExpirationMs());
    assertEquals(defaultPartitionLifetimeSec * 1000L, cloudDataset.getDefaultPartitionExpirationMs());
    assertEquals(resource, controlledResourceService.getControlledResource(workspace.getWorkspaceId(), resource.getResourceId(), user.getAuthenticatedRequest()));
}
Also used : HashMap(java.util.HashMap) Dataset(com.google.api.services.bigquery.model.Dataset) ApiGcpBigQueryDatasetCreationParameters(bio.terra.workspace.generated.model.ApiGcpBigQueryDatasetCreationParameters) StepStatus(bio.terra.stairway.StepStatus) ControlledBigQueryDatasetResource(bio.terra.workspace.service.resource.controlled.cloud.gcp.bqdataset.ControlledBigQueryDatasetResource) BigQueryCow(bio.terra.cloudres.google.bigquery.BigQueryCow) CreateBigQueryDatasetStep(bio.terra.workspace.service.resource.controlled.cloud.gcp.bqdataset.CreateBigQueryDatasetStep) Test(org.junit.jupiter.api.Test) BaseConnectedTest(bio.terra.workspace.common.BaseConnectedTest) DisabledIfEnvironmentVariable(org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable)

Example 2 with BigQueryCow

use of bio.terra.cloudres.google.bigquery.BigQueryCow in project terra-workspace-manager by DataBiosphere.

the class ControlledResourceServiceTest method validateBigQueryDatasetCloudMetadata.

/**
 * Lookup the location and expiration times stored on the cloud for a BigQuery dataset, and assert
 * they match the given values.
 */
private void validateBigQueryDatasetCloudMetadata(String projectId, String datasetId, String location, Integer defaultTableExpirationSec, Integer defaultPartitionExpirationSec) throws IOException {
    BigQueryCow bqCow = crlService.createWsmSaBigQueryCow();
    Dataset cloudDataset = bqCow.datasets().get(projectId, datasetId).execute();
    assertEquals(location, cloudDataset.getLocation());
    if (defaultTableExpirationSec == null) {
        assertNull(cloudDataset.getDefaultTableExpirationMs());
    } else {
        assertEquals(defaultTableExpirationSec * 1000, cloudDataset.getDefaultTableExpirationMs());
    }
    if (defaultPartitionExpirationSec == null) {
        assertNull(cloudDataset.getDefaultPartitionExpirationMs());
    } else {
        assertEquals(defaultPartitionExpirationSec * 1000, cloudDataset.getDefaultPartitionExpirationMs());
    }
}
Also used : Dataset(com.google.api.services.bigquery.model.Dataset) BigQueryCow(bio.terra.cloudres.google.bigquery.BigQueryCow)

Example 3 with BigQueryCow

use of bio.terra.cloudres.google.bigquery.BigQueryCow in project terra-workspace-manager by DataBiosphere.

the class ControlledResourceServiceTest method deleteBqDatasetUndo.

@Test
@DisabledIfEnvironmentVariable(named = "TEST_ENV", matches = BUFFER_SERVICE_DISABLED_ENVS_REG_EX)
void deleteBqDatasetUndo() throws Exception {
    String datasetId = ControlledResourceFixtures.uniqueDatasetId();
    String location = "us-central1";
    ApiGcpBigQueryDatasetCreationParameters creationParameters = new ApiGcpBigQueryDatasetCreationParameters().datasetId(datasetId).location(location);
    ControlledBigQueryDatasetResource resource = ControlledResourceFixtures.makeDefaultControlledBigQueryBuilder(workspace.getWorkspaceId()).datasetName(datasetId).build();
    ControlledBigQueryDatasetResource createdDataset = controlledResourceService.createControlledResourceSync(resource, null, user.getAuthenticatedRequest(), creationParameters).castByEnum(WsmResourceType.CONTROLLED_GCP_BIG_QUERY_DATASET);
    assertEquals(resource, createdDataset);
    // None of the steps on this flight are undoable, so even with lastStepFailure set to true we
    // should expect the resource to really be deleted.
    jobService.setFlightDebugInfoForTest(FlightDebugInfo.newBuilder().lastStepFailure(true).build());
    assertThrows(InvalidResultStateException.class, () -> controlledResourceService.deleteControlledResourceSync(resource.getWorkspaceId(), resource.getResourceId(), user.getAuthenticatedRequest()));
    BigQueryCow bqCow = crlService.createWsmSaBigQueryCow();
    GoogleJsonResponseException getException = assertThrows(GoogleJsonResponseException.class, () -> bqCow.datasets().get(projectId, resource.getDatasetName()).execute());
    assertEquals(HttpStatus.NOT_FOUND.value(), getException.getStatusCode());
    assertThrows(ResourceNotFoundException.class, () -> controlledResourceService.getControlledResource(workspace.getWorkspaceId(), resource.getResourceId(), user.getAuthenticatedRequest()));
}
Also used : GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) ApiGcpBigQueryDatasetCreationParameters(bio.terra.workspace.generated.model.ApiGcpBigQueryDatasetCreationParameters) ControlledBigQueryDatasetResource(bio.terra.workspace.service.resource.controlled.cloud.gcp.bqdataset.ControlledBigQueryDatasetResource) BigQueryCow(bio.terra.cloudres.google.bigquery.BigQueryCow) Test(org.junit.jupiter.api.Test) BaseConnectedTest(bio.terra.workspace.common.BaseConnectedTest) DisabledIfEnvironmentVariable(org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable)

Example 4 with BigQueryCow

use of bio.terra.cloudres.google.bigquery.BigQueryCow in project terra-workspace-manager by DataBiosphere.

the class DeleteBigQueryDatasetStep method doStep.

@Override
public StepResult doStep(FlightContext flightContext) throws InterruptedException, RetryException {
    final GcpCloudContext gcpCloudContext = flightContext.getWorkingMap().get(ControlledResourceKeys.GCP_CLOUD_CONTEXT, GcpCloudContext.class);
    String projectId = gcpCloudContext.getGcpProjectId();
    BigQueryCow bqCow = crlService.createWsmSaBigQueryCow();
    try {
        // With deleteContents set to true, this will delete the dataset even if it still has tables.
        bqCow.datasets().delete(projectId, resource.getDatasetName()).setDeleteContents(true).execute();
    } catch (GoogleJsonResponseException e) {
        if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
            logger.info("BQ dataset {} in project {} already deleted", resource.getDatasetName(), projectId);
            return StepResult.getStepResultSuccess();
        }
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
    } catch (IOException e) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
    }
    return StepResult.getStepResultSuccess();
}
Also used : GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) IOException(java.io.IOException) StepResult(bio.terra.stairway.StepResult) BigQueryCow(bio.terra.cloudres.google.bigquery.BigQueryCow) GcpCloudContext(bio.terra.workspace.service.workspace.model.GcpCloudContext)

Example 5 with BigQueryCow

use of bio.terra.cloudres.google.bigquery.BigQueryCow in project terra-workspace-manager by DataBiosphere.

the class CreateBigQueryDatasetStep method undoStep.

@Override
public StepResult undoStep(FlightContext flightContext) throws InterruptedException {
    String projectId = gcpCloudContextService.getRequiredGcpProject(resource.getWorkspaceId());
    BigQueryCow bqCow = crlService.createWsmSaBigQueryCow();
    try {
        // With deleteContents set to true, this will delete the dataset even if other steps fail
        // to clean up tables or data.
        bqCow.datasets().delete(projectId, resource.getDatasetName()).setDeleteContents(true).execute();
    } catch (GoogleJsonResponseException e) {
        // Stairway steps may run multiple times, so we may already have deleted this resource.
        if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
            logger.info("BQ dataset {} in project {} already deleted", resource.getDatasetName(), projectId);
            return StepResult.getStepResultSuccess();
        }
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
    } catch (IOException e) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e);
    }
    return StepResult.getStepResultSuccess();
}
Also used : GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) IOException(java.io.IOException) StepResult(bio.terra.stairway.StepResult) BigQueryCow(bio.terra.cloudres.google.bigquery.BigQueryCow)

Aggregations

BigQueryCow (bio.terra.cloudres.google.bigquery.BigQueryCow)14 StepResult (bio.terra.stairway.StepResult)7 Dataset (com.google.api.services.bigquery.model.Dataset)7 IOException (java.io.IOException)7 GoogleJsonResponseException (com.google.api.client.googleapis.json.GoogleJsonResponseException)6 ApiGcpBigQueryDatasetCreationParameters (bio.terra.workspace.generated.model.ApiGcpBigQueryDatasetCreationParameters)5 ControlledBigQueryDatasetResource (bio.terra.workspace.service.resource.controlled.cloud.gcp.bqdataset.ControlledBigQueryDatasetResource)5 StepStatus (bio.terra.stairway.StepStatus)4 BaseConnectedTest (bio.terra.workspace.common.BaseConnectedTest)4 HashMap (java.util.HashMap)4 Test (org.junit.jupiter.api.Test)4 DisabledIfEnvironmentVariable (org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable)4 FlightMap (bio.terra.stairway.FlightMap)3 CreateBigQueryDatasetStep (bio.terra.workspace.service.resource.controlled.cloud.gcp.bqdataset.CreateBigQueryDatasetStep)2 GcpCloudContext (bio.terra.workspace.service.workspace.model.GcpCloudContext)2 FlightContext (bio.terra.stairway.FlightContext)1 Step (bio.terra.stairway.Step)1 RetryException (bio.terra.stairway.exception.RetryException)1 ApiGcpBigQueryDatasetUpdateParameters (bio.terra.workspace.generated.model.ApiGcpBigQueryDatasetUpdateParameters)1 CrlService (bio.terra.workspace.service.crl.CrlService)1