Search in sources :

Example 1 with RegionName

use of com.google.cloud.dataproc.v1.RegionName in project java-dataproc by googleapis.

the class WorkflowTemplateServiceClientTest method listWorkflowTemplatesTest2.

@Test
public void listWorkflowTemplatesTest2() throws Exception {
    WorkflowTemplate responsesElement = WorkflowTemplate.newBuilder().build();
    ListWorkflowTemplatesResponse expectedResponse = ListWorkflowTemplatesResponse.newBuilder().setNextPageToken("").addAllTemplates(Arrays.asList(responsesElement)).build();
    mockWorkflowTemplateService.addResponse(expectedResponse);
    RegionName parent = RegionName.of("[PROJECT]", "[REGION]");
    ListWorkflowTemplatesPagedResponse pagedListResponse = client.listWorkflowTemplates(parent);
    List<WorkflowTemplate> resources = Lists.newArrayList(pagedListResponse.iterateAll());
    Assert.assertEquals(1, resources.size());
    Assert.assertEquals(expectedResponse.getTemplatesList().get(0), resources.get(0));
    List<AbstractMessage> actualRequests = mockWorkflowTemplateService.getRequests();
    Assert.assertEquals(1, actualRequests.size());
    ListWorkflowTemplatesRequest actualRequest = ((ListWorkflowTemplatesRequest) actualRequests.get(0));
    Assert.assertEquals(parent.toString(), actualRequest.getParent());
    Assert.assertTrue(channelProvider.isHeaderSent(ApiClientHeaderProvider.getDefaultApiClientHeaderKey(), GaxGrpcProperties.getDefaultApiClientHeaderPattern()));
}
Also used : ListWorkflowTemplatesPagedResponse(com.google.cloud.dataproc.v1.WorkflowTemplateServiceClient.ListWorkflowTemplatesPagedResponse) AbstractMessage(com.google.protobuf.AbstractMessage) Test(org.junit.Test)

Example 2 with RegionName

use of com.google.cloud.dataproc.v1.RegionName in project java-dataproc by googleapis.

the class AutoscalingPolicyServiceClientTest method listAutoscalingPoliciesTest2.

@Test
public void listAutoscalingPoliciesTest2() throws Exception {
    AutoscalingPolicy responsesElement = AutoscalingPolicy.newBuilder().build();
    ListAutoscalingPoliciesResponse expectedResponse = ListAutoscalingPoliciesResponse.newBuilder().setNextPageToken("").addAllPolicies(Arrays.asList(responsesElement)).build();
    mockAutoscalingPolicyService.addResponse(expectedResponse);
    RegionName parent = RegionName.of("[PROJECT]", "[REGION]");
    ListAutoscalingPoliciesPagedResponse pagedListResponse = client.listAutoscalingPolicies(parent);
    List<AutoscalingPolicy> resources = Lists.newArrayList(pagedListResponse.iterateAll());
    Assert.assertEquals(1, resources.size());
    Assert.assertEquals(expectedResponse.getPoliciesList().get(0), resources.get(0));
    List<AbstractMessage> actualRequests = mockAutoscalingPolicyService.getRequests();
    Assert.assertEquals(1, actualRequests.size());
    ListAutoscalingPoliciesRequest actualRequest = ((ListAutoscalingPoliciesRequest) actualRequests.get(0));
    Assert.assertEquals(parent.toString(), actualRequest.getParent());
    Assert.assertTrue(channelProvider.isHeaderSent(ApiClientHeaderProvider.getDefaultApiClientHeaderKey(), GaxGrpcProperties.getDefaultApiClientHeaderPattern()));
}
Also used : AbstractMessage(com.google.protobuf.AbstractMessage) ListAutoscalingPoliciesPagedResponse(com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient.ListAutoscalingPoliciesPagedResponse) Test(org.junit.Test)

Example 3 with RegionName

use of com.google.cloud.dataproc.v1.RegionName in project cdap by caskdata.

the class PredefinedAutoScaling method createPredefinedAutoScalingPolicy.

// Creates the auto-scaling policy if doesn't exist.
public String createPredefinedAutoScalingPolicy() throws IOException {
    AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), AUTOSCALING_POLICY_ID);
    AutoscalingPolicy generatedPolicy = generatePredefinedAutoScaling();
    try (AutoscalingPolicyServiceClient autoscalingPolicyServiceClient = getAutoscalingPolicyServiceClient()) {
        // If the policy already exists meaning it is already created by cdap or in rare scenario the customer might have
        // created with a similar name. It is also possible that the user might have changed configs in the autoscaling
        // policy.
        // So, if exists -> we will fetch and check if it differs from our predefined values and LOG it
        AutoscalingPolicy existingPolicy = null;
        boolean createPolicy = false;
        try {
            existingPolicy = autoscalingPolicyServiceClient.getAutoscalingPolicy(autoscalingPolicyName);
            boolean yarnDiff = !existingPolicy.getBasicAlgorithm().getYarnConfig().equals(generatedPolicy.getBasicAlgorithm().getYarnConfig());
            boolean workerDiff = !existingPolicy.getWorkerConfig().equals(generatedPolicy.getWorkerConfig());
            boolean secondaryWorkerDiff = !existingPolicy.getSecondaryWorkerConfig().equals(generatedPolicy.getSecondaryWorkerConfig());
            if (yarnDiff || workerDiff || secondaryWorkerDiff) {
                LOG.warn("The predefined auto-scaling policy {} already exists and is having a different configuration" + "as compared to CDF/CDAP's chosen configuration", existingPolicy.getName());
            }
        } catch (NotFoundException e) {
            createPolicy = true;
            LOG.debug("The autoscaling policy doesn't exists");
        }
        if (createPolicy) {
            RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
            try {
                autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, generatedPolicy);
            } catch (AlreadyExistsException e) {
            // no - op Race condition: in case 2 pipelines with auto-scaling are triggered together and might try to
            // create in parallel
            }
        }
    }
    return autoscalingPolicyName.toString();
}
Also used : AutoscalingPolicy(com.google.cloud.dataproc.v1.AutoscalingPolicy) AlreadyExistsException(com.google.api.gax.rpc.AlreadyExistsException) RegionName(com.google.cloud.dataproc.v1.RegionName) NotFoundException(com.google.api.gax.rpc.NotFoundException) AutoscalingPolicyName(com.google.cloud.dataproc.v1.AutoscalingPolicyName) AutoscalingPolicyServiceClient(com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient)

Example 4 with RegionName

use of com.google.cloud.dataproc.v1.RegionName in project cdap by caskdata.

the class PredefinedAutoScalingTest method testFetchFailedAndCreateIsCalled.

@Test
public void testFetchFailedAndCreateIsCalled() throws IOException {
    PredefinedAutoScaling predefinedAutoScaling = new PredefinedAutoScaling(dataprocConf);
    // mock Return generated auto-scaling policy while fetching
    AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), PredefinedAutoScaling.AUTOSCALING_POLICY_ID);
    AutoscalingPolicy generatedPolicy = predefinedAutoScaling.generatePredefinedAutoScaling();
    AutoscalingPolicyServiceClient mockClient = PowerMockito.mock(AutoscalingPolicyServiceClient.class);
    Mockito.when(mockClient.getAutoscalingPolicy(autoscalingPolicyName)).thenThrow(NotFoundException.class);
    RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
    Mockito.when(mockClient.createAutoscalingPolicy(parent, generatedPolicy)).thenReturn(null);
    PredefinedAutoScaling spy = Mockito.spy(predefinedAutoScaling);
    Mockito.doReturn(mockClient).when(spy).getAutoscalingPolicyServiceClient();
    String name = spy.createPredefinedAutoScalingPolicy();
    Mockito.verify(mockClient, Mockito.times(1)).getAutoscalingPolicy(autoscalingPolicyName);
    // verify that create call is not made
    Mockito.verify(mockClient, Mockito.times(1)).createAutoscalingPolicy(parent, generatedPolicy);
    Assert.assertEquals(name, autoscalingPolicyName.toString());
}
Also used : AutoscalingPolicy(com.google.cloud.dataproc.v1.AutoscalingPolicy) RegionName(com.google.cloud.dataproc.v1.RegionName) AutoscalingPolicyName(com.google.cloud.dataproc.v1.AutoscalingPolicyName) AutoscalingPolicyServiceClient(com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient) Test(org.junit.Test) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest)

Example 5 with RegionName

use of com.google.cloud.dataproc.v1.RegionName in project cdap by caskdata.

the class PredefinedAutoScalingTest method testFetchingExistingAutoScalingPolicySuccess.

@Test
public void testFetchingExistingAutoScalingPolicySuccess() throws IOException {
    PredefinedAutoScaling predefinedAutoScaling = new PredefinedAutoScaling(dataprocConf);
    // mock Return generated auto-scaling policy while fetching
    AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), PredefinedAutoScaling.AUTOSCALING_POLICY_ID);
    AutoscalingPolicy generatedPolicy = predefinedAutoScaling.generatePredefinedAutoScaling();
    AutoscalingPolicyServiceClient mockClient = PowerMockito.mock(AutoscalingPolicyServiceClient.class);
    Mockito.when(mockClient.getAutoscalingPolicy(autoscalingPolicyName)).thenReturn(generatedPolicy);
    PredefinedAutoScaling spy = Mockito.spy(predefinedAutoScaling);
    Mockito.doReturn(mockClient).when(spy).getAutoscalingPolicyServiceClient();
    String name = spy.createPredefinedAutoScalingPolicy();
    Mockito.verify(mockClient, Mockito.times(1)).getAutoscalingPolicy(autoscalingPolicyName);
    // verify that create call is not made
    RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
    Mockito.verify(mockClient, Mockito.never()).createAutoscalingPolicy(parent, generatedPolicy);
    Assert.assertEquals(name, autoscalingPolicyName.toString());
}
Also used : AutoscalingPolicy(com.google.cloud.dataproc.v1.AutoscalingPolicy) RegionName(com.google.cloud.dataproc.v1.RegionName) AutoscalingPolicyName(com.google.cloud.dataproc.v1.AutoscalingPolicyName) AutoscalingPolicyServiceClient(com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient) Test(org.junit.Test) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest)

Aggregations

AutoscalingPolicy (com.google.cloud.dataproc.v1.AutoscalingPolicy)7 AutoscalingPolicyServiceClient (com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient)7 RegionName (com.google.cloud.dataproc.v1.RegionName)7 AutoscalingPolicyName (com.google.cloud.dataproc.v1.AutoscalingPolicyName)6 Test (org.junit.Test)6 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)4 AlreadyExistsException (com.google.api.gax.rpc.AlreadyExistsException)2 NotFoundException (com.google.api.gax.rpc.NotFoundException)2 AbstractMessage (com.google.protobuf.AbstractMessage)2 AutoscalingConfig (com.google.cloud.dataproc.v1.AutoscalingConfig)1 ListAutoscalingPoliciesPagedResponse (com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient.ListAutoscalingPoliciesPagedResponse)1 AutoscalingPolicyServiceSettings (com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings)1 BasicAutoscalingAlgorithm (com.google.cloud.dataproc.v1.BasicAutoscalingAlgorithm)1 BasicYarnAutoscalingConfig (com.google.cloud.dataproc.v1.BasicYarnAutoscalingConfig)1 Cluster (com.google.cloud.dataproc.v1.Cluster)1 ClusterConfig (com.google.cloud.dataproc.v1.ClusterConfig)1 ClusterControllerClient (com.google.cloud.dataproc.v1.ClusterControllerClient)1 ClusterControllerSettings (com.google.cloud.dataproc.v1.ClusterControllerSettings)1 ClusterOperationMetadata (com.google.cloud.dataproc.v1.ClusterOperationMetadata)1 InstanceGroupAutoscalingPolicyConfig (com.google.cloud.dataproc.v1.InstanceGroupAutoscalingPolicyConfig)1