Search in sources :

Example 1 with ServiceUsage

use of com.google.api.services.serviceusage.v1beta1.ServiceUsage in project terra-resource-buffer by DataBiosphere.

the class CreateProjectFlightIntegrationTest method testCreateGoogleProject_createsConsumerOverride.

@Test
public void testCreateGoogleProject_createsConsumerOverride() throws Exception {
    FlightManager manager = new FlightManager(bufferDao, new StubSubmissionFlightFactory(LatchBeforeAssertResourceStep.class), stairwayComponent, transactionTemplate);
    GcpProjectConfig gcpProjectConfig = newBasicGcpConfig().serviceUsage(new ServiceUsage().bigQuery(new BigQueryQuotas().overrideBigQueryDailyUsageQuota(true).bigQueryDailyUsageQuotaOverrideValueMebibytes(new BigDecimal(CONSUMER_QUOTA_OVERRIDE_VALUE_MEBIBYTES))));
    Pool pool = preparePool(bufferDao, gcpProjectConfig);
    String flightId = manager.submitCreationFlight(pool).orElseThrow();
    ResourceId resourceId = extractResourceIdFromFlightState(blockUntilFlightComplete(stairwayComponent, flightId));
    Project project = assertProjectExists(resourceId);
    String projectNumber = getProjectNumberFromName(project.getName());
    String parent = String.format("projects/%s/services/bigquery.googleapis.com/consumerQuotaMetrics/" + "bigquery.googleapis.com%%2Fquota%%2Fquery%%2Fusage/limits/%%2Fd%%2Fproject", projectNumber);
    ServiceUsageCow.Services.ConsumerQuotaMetrics.Limits.ConsumerOverrides.List list = serviceUsageCow.services().consumerQuotaMetrics().limits().consumerOverrides().list(parent);
    ListConsumerOverridesResponse response = list.execute();
    assertEquals(1, response.getOverrides().size(), "single override expected");
    QuotaOverride quotaOverride = response.getOverrides().get(0);
    assertEquals(CONSUMER_QUOTA_OVERRIDE_VALUE_MEBIBYTES, quotaOverride.getOverrideValue());
}
Also used : ListConsumerOverridesResponse(com.google.api.services.serviceusage.v1beta1.model.ListConsumerOverridesResponse) FlightManager(bio.terra.buffer.service.resource.FlightManager) StubSubmissionFlightFactory(bio.terra.buffer.integration.IntegrationUtils.StubSubmissionFlightFactory) BigDecimal(java.math.BigDecimal) Project(com.google.api.services.cloudresourcemanager.v3.model.Project) QuotaOverride(com.google.api.services.serviceusage.v1beta1.model.QuotaOverride) ResourceId(bio.terra.buffer.common.ResourceId) Pool(bio.terra.buffer.common.Pool) IntegrationUtils.preparePool(bio.terra.buffer.integration.IntegrationUtils.preparePool) ServiceUsageCow(bio.terra.cloudres.google.serviceusage.ServiceUsageCow) Test(org.junit.jupiter.api.Test) BaseIntegrationTest(bio.terra.buffer.common.BaseIntegrationTest)

Example 2 with ServiceUsage

use of com.google.api.services.serviceusage.v1beta1.ServiceUsage in project terra-cloud-resource-lib by DataBiosphere.

the class ServiceUsageCowTest method listAndEnableServices.

@Test
public void listAndEnableServices() throws Exception {
    ServiceUsageCow serviceUsage = defaultServiceUsage();
    Project project = ProjectUtils.executeCreateProject();
    String projectName = projectIdToName(project.getProjectId());
    String storageServiceName = serviceName(project, STORAGE_SERVICE_ID);
    ListServicesResponse response1 = serviceUsage.services().list(projectName).setFilter(ENABLED_FILTER).execute();
    assertNull(response1.getServices());
    Operation operation = serviceUsage.services().batchEnable(projectName, new BatchEnableServicesRequest().setServiceIds(ImmutableList.of(STORAGE_SERVICE_ID))).execute();
    OperationTestUtils.pollAndAssertSuccess(serviceUsage.operations().operationCow(operation), Duration.ofSeconds(5), Duration.ofSeconds(60));
    ListServicesResponse response2 = serviceUsage.services().list(projectName).setFilter(ENABLED_FILTER).execute();
    List<String> services2 = response2.getServices().stream().map(Service::getName).collect(Collectors.toList());
    assertThat(services2, Matchers.hasItem(storageServiceName));
}
Also used : Project(com.google.api.services.cloudresourcemanager.v3.model.Project) ListServicesResponse(com.google.api.services.serviceusage.v1beta1.model.ListServicesResponse) Operation(com.google.api.services.serviceusage.v1beta1.model.Operation) BatchEnableServicesRequest(com.google.api.services.serviceusage.v1beta1.model.BatchEnableServicesRequest) Test(org.junit.jupiter.api.Test)

Example 3 with ServiceUsage

use of com.google.api.services.serviceusage.v1beta1.ServiceUsage in project pipeline5 by hartwigmedical.

the class QuotaConstrainedComputeEngineTest method ensuresVmsDontExceedMaxCPUForRegion.

@Test
public void ensuresVmsDontExceedMaxCPUForRegion() throws Exception {
    ComputeEngine decorated = mock(ComputeEngine.class);
    ServiceUsage serviceUsage = mock(ServiceUsage.class);
    ServiceUsage.Services services = mock(ServiceUsage.Services.class);
    ServiceUsage.Services.ConsumerQuotaMetrics consumerQuotaMetrics = mock(ServiceUsage.Services.ConsumerQuotaMetrics.class);
    ServiceUsage.Services.ConsumerQuotaMetrics.Limits limits = mock(ServiceUsage.Services.ConsumerQuotaMetrics.Limits.class);
    ServiceUsage.Services.ConsumerQuotaMetrics.Limits.Get limitsGet = mock(ServiceUsage.Services.ConsumerQuotaMetrics.Limits.Get.class);
    when(serviceUsage.services()).thenReturn(services);
    when(services.consumerQuotaMetrics()).thenReturn(consumerQuotaMetrics);
    when(consumerQuotaMetrics.limits()).thenReturn(limits);
    ArgumentCaptor<String> quotaName = ArgumentCaptor.forClass(String.class);
    when(limits.get(quotaName.capture())).thenReturn(limitsGet);
    ConsumerQuotaLimit limit = new ConsumerQuotaLimit().setQuotaBuckets(List.of(new QuotaBucket().setEffectiveLimit(10L).setDimensions(Map.of(REGION, REGION))));
    when(limitsGet.execute()).thenReturn(limit);
    VirtualMachineJobDefinition jobDefinition = VirtualMachineJobDefinition.builder().name("test").namespacedResults(ResultsDirectory.defaultDirectory()).startupCommand(BashStartupScript.of("empty")).performanceProfile(VirtualMachinePerformanceProfile.custom(10, 10)).build();
    ArgumentCaptor<VirtualMachineJobDefinition> constrained = ArgumentCaptor.forClass(VirtualMachineJobDefinition.class);
    when(decorated.submit(any(), constrained.capture(), any())).thenReturn(PipelineStatus.SUCCESS);
    QuotaConstrainedComputeEngine victim = new QuotaConstrainedComputeEngine(decorated, serviceUsage, REGION, PROJECT, 0.6);
    PipelineStatus result = victim.submit(MockRuntimeBucket.test().getRuntimeBucket(), jobDefinition);
    assertThat(result).isEqualTo(PipelineStatus.SUCCESS);
    MachineType machineType = constrained.getValue().performanceProfile().machineType();
    assertThat(machineType.cpus()).isEqualTo(6);
    assertThat(machineType.memoryGB()).isEqualTo(6);
}
Also used : PipelineStatus(com.hartwig.pipeline.execution.PipelineStatus) MachineType(com.hartwig.pipeline.execution.MachineType) ServiceUsage(com.google.api.services.serviceusage.v1beta1.ServiceUsage) ConsumerQuotaLimit(com.google.api.services.serviceusage.v1beta1.model.ConsumerQuotaLimit) QuotaBucket(com.google.api.services.serviceusage.v1beta1.model.QuotaBucket) Test(org.junit.Test)

Example 4 with ServiceUsage

use of com.google.api.services.serviceusage.v1beta1.ServiceUsage in project jade-data-repo by DataBiosphere.

the class GoogleResourceService method enableServices.

private void enableServices(GoogleProjectResource projectResource) throws InterruptedException {
    BatchEnableServicesRequest batchRequest = new BatchEnableServicesRequest().setServiceIds(projectResource.getServiceIds());
    try {
        ServiceUsage serviceUsage = serviceUsage();
        String projectNumberString = "projects/" + projectResource.getGoogleProjectNumber();
        logger.info("trying to get services for {} ({})", projectNumberString, projectResource.getGoogleProjectId());
        ServiceUsage.Services.List list = serviceUsage.services().list(projectNumberString).setFilter(ENABLED_FILTER);
        ListServicesResponse listServicesResponse = list.execute();
        logger.info("found: " + String.join(", ", projectResource.getServiceIds()));
        List<String> services = projectResource.getServiceIds().stream().map(s -> String.format("%s/services/%s", projectNumberString, s)).collect(Collectors.toList());
        List<Service> serviceList = listServicesResponse.getServices();
        List<String> actualServiceNames = Collections.emptyList();
        if (serviceList != null) {
            actualServiceNames = serviceList.stream().map(s -> s.getName()).collect(Collectors.toList());
        }
        if (actualServiceNames.containsAll(services)) {
            logger.info("project already has the right resources enabled, skipping");
        } else {
            logger.info("project does not have all resources enabled");
            ServiceUsage.Services.BatchEnable batchEnable = serviceUsage.services().batchEnable(projectNumberString, batchRequest);
            long timeout = resourceConfiguration.getProjectCreateTimeoutSeconds();
            blockUntilServiceOperationComplete(serviceUsage, batchEnable.execute(), timeout);
        }
    } catch (IOException | GeneralSecurityException e) {
        throw new GoogleResourceException("Could not enable services", e);
    }
}
Also used : StringUtils(org.apache.commons.lang.StringUtils) Status(com.google.api.services.cloudresourcemanager.model.Status) ServiceUsage(com.google.api.services.serviceusage.v1beta1.ServiceUsage) ListServicesResponse(com.google.api.services.serviceusage.v1beta1.model.ListServicesResponse) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) StorageOptions(com.google.cloud.storage.StorageOptions) Binding(com.google.api.services.cloudresourcemanager.model.Binding) GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) GeneralSecurityException(java.security.GeneralSecurityException) InaccessibleBillingAccountException(bio.terra.service.resourcemanagement.exception.InaccessibleBillingAccountException) Map(java.util.Map) Bucket(com.google.cloud.storage.Bucket) Project(com.google.api.services.cloudresourcemanager.model.Project) JacksonFactory(com.google.api.client.json.jackson2.JacksonFactory) BucketInfo(com.google.cloud.storage.BucketInfo) Collection(java.util.Collection) HttpTransport(com.google.api.client.http.HttpTransport) GoogleResourceException(bio.terra.service.resourcemanagement.exception.GoogleResourceException) GoogleNetHttpTransport(com.google.api.client.googleapis.javanet.GoogleNetHttpTransport) UUID(java.util.UUID) BatchEnableServicesRequest(com.google.api.services.serviceusage.v1beta1.model.BatchEnableServicesRequest) Collectors(java.util.stream.Collectors) GcsProjectFactory(bio.terra.service.filedata.google.gcs.GcsProjectFactory) CorruptMetadataException(bio.terra.service.snapshot.exception.CorruptMetadataException) ConfigEnum(bio.terra.service.configuration.ConfigEnum) List(java.util.List) Service(com.google.api.services.serviceusage.v1beta1.model.Service) Environment(org.springframework.core.env.Environment) Storage(com.google.cloud.storage.Storage) ConfigurationService(bio.terra.service.configuration.ConfigurationService) GoogleCredential(com.google.api.client.googleapis.auth.oauth2.GoogleCredential) SetIamPolicyRequest(com.google.api.services.cloudresourcemanager.model.SetIamPolicyRequest) HashMap(java.util.HashMap) GetIamPolicyRequest(com.google.api.services.cloudresourcemanager.model.GetIamPolicyRequest) Value(org.springframework.beans.factory.annotation.Value) CloudResourceManager(com.google.api.services.cloudresourcemanager.CloudResourceManager) ResourceId(com.google.api.services.cloudresourcemanager.model.ResourceId) EnablePermissionsFailedException(bio.terra.service.resourcemanagement.exception.EnablePermissionsFailedException) BillingProfile(bio.terra.service.resourcemanagement.BillingProfile) BucketLockException(bio.terra.service.resourcemanagement.exception.BucketLockException) Logger(org.slf4j.Logger) GcsProject(bio.terra.service.filedata.google.gcs.GcsProject) Policy(com.google.api.services.cloudresourcemanager.model.Policy) ProfileService(bio.terra.service.resourcemanagement.ProfileService) Acl(com.google.cloud.storage.Acl) IOException(java.io.IOException) TimeUnit(java.util.concurrent.TimeUnit) Component(org.springframework.stereotype.Component) JsonFactory(com.google.api.client.json.JsonFactory) StorageException(com.google.cloud.storage.StorageException) Operation(com.google.api.services.cloudresourcemanager.model.Operation) GoogleResourceNotFoundException(bio.terra.service.resourcemanagement.exception.GoogleResourceNotFoundException) StorageClass(com.google.cloud.storage.StorageClass) Collections(java.util.Collections) GeneralSecurityException(java.security.GeneralSecurityException) Service(com.google.api.services.serviceusage.v1beta1.model.Service) ConfigurationService(bio.terra.service.configuration.ConfigurationService) ProfileService(bio.terra.service.resourcemanagement.ProfileService) IOException(java.io.IOException) ListServicesResponse(com.google.api.services.serviceusage.v1beta1.model.ListServicesResponse) ServiceUsage(com.google.api.services.serviceusage.v1beta1.ServiceUsage) BatchEnableServicesRequest(com.google.api.services.serviceusage.v1beta1.model.BatchEnableServicesRequest) GoogleResourceException(bio.terra.service.resourcemanagement.exception.GoogleResourceException)

Aggregations

Project (com.google.api.services.cloudresourcemanager.v3.model.Project)2 ServiceUsage (com.google.api.services.serviceusage.v1beta1.ServiceUsage)2 BatchEnableServicesRequest (com.google.api.services.serviceusage.v1beta1.model.BatchEnableServicesRequest)2 ListServicesResponse (com.google.api.services.serviceusage.v1beta1.model.ListServicesResponse)2 Test (org.junit.jupiter.api.Test)2 BaseIntegrationTest (bio.terra.buffer.common.BaseIntegrationTest)1 Pool (bio.terra.buffer.common.Pool)1 ResourceId (bio.terra.buffer.common.ResourceId)1 StubSubmissionFlightFactory (bio.terra.buffer.integration.IntegrationUtils.StubSubmissionFlightFactory)1 IntegrationUtils.preparePool (bio.terra.buffer.integration.IntegrationUtils.preparePool)1 FlightManager (bio.terra.buffer.service.resource.FlightManager)1 ServiceUsageCow (bio.terra.cloudres.google.serviceusage.ServiceUsageCow)1 ConfigEnum (bio.terra.service.configuration.ConfigEnum)1 ConfigurationService (bio.terra.service.configuration.ConfigurationService)1 GcsProject (bio.terra.service.filedata.google.gcs.GcsProject)1 GcsProjectFactory (bio.terra.service.filedata.google.gcs.GcsProjectFactory)1 BillingProfile (bio.terra.service.resourcemanagement.BillingProfile)1 ProfileService (bio.terra.service.resourcemanagement.ProfileService)1 BucketLockException (bio.terra.service.resourcemanagement.exception.BucketLockException)1 EnablePermissionsFailedException (bio.terra.service.resourcemanagement.exception.EnablePermissionsFailedException)1