Examples with AutoscalingPolicyServiceClient com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient used on opensource projects

Example 6 with AutoscalingPolicyServiceClient

use of com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient in project cdap by caskdata.

the class PredefinedAutoScalingTest method testFetchingExistingAutoScalingPolicySuccess.

@Test
public void testFetchingExistingAutoScalingPolicySuccess() throws IOException {
    PredefinedAutoScaling predefinedAutoScaling = new PredefinedAutoScaling(dataprocConf);
    // mock Return generated auto-scaling policy while fetching
    AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), PredefinedAutoScaling.AUTOSCALING_POLICY_ID);
    AutoscalingPolicy generatedPolicy = predefinedAutoScaling.generatePredefinedAutoScaling();
    AutoscalingPolicyServiceClient mockClient = PowerMockito.mock(AutoscalingPolicyServiceClient.class);
    Mockito.when(mockClient.getAutoscalingPolicy(autoscalingPolicyName)).thenReturn(generatedPolicy);
    PredefinedAutoScaling spy = Mockito.spy(predefinedAutoScaling);
    Mockito.doReturn(mockClient).when(spy).getAutoscalingPolicyServiceClient();
    String name = spy.createPredefinedAutoScalingPolicy();
    Mockito.verify(mockClient, Mockito.times(1)).getAutoscalingPolicy(autoscalingPolicyName);
    // verify that create call is not made
    RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
    Mockito.verify(mockClient, Mockito.never()).createAutoscalingPolicy(parent, generatedPolicy);
    Assert.assertEquals(name, autoscalingPolicyName.toString());
}

Example 7 with AutoscalingPolicyServiceClient

use of com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient in project cdap by cdapio.

the class PredefinedAutoScaling method createPredefinedAutoScalingPolicy.

// Creates the auto-scaling policy if doesn't exist.
public String createPredefinedAutoScalingPolicy() throws IOException {
    AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), AUTOSCALING_POLICY_ID);
    AutoscalingPolicy generatedPolicy = generatePredefinedAutoScaling();
    try (AutoscalingPolicyServiceClient autoscalingPolicyServiceClient = getAutoscalingPolicyServiceClient()) {
        // If the policy already exists meaning it is already created by cdap or in rare scenario the customer might have
        // created with a similar name. It is also possible that the user might have changed configs in the autoscaling
        // policy.
        // So, if exists -> we will fetch and check if it differs from our predefined values and LOG it
        AutoscalingPolicy existingPolicy = null;
        boolean createPolicy = false;
        try {
            existingPolicy = autoscalingPolicyServiceClient.getAutoscalingPolicy(autoscalingPolicyName);
            boolean yarnDiff = !existingPolicy.getBasicAlgorithm().getYarnConfig().equals(generatedPolicy.getBasicAlgorithm().getYarnConfig());
            boolean workerDiff = !existingPolicy.getWorkerConfig().equals(generatedPolicy.getWorkerConfig());
            boolean secondaryWorkerDiff = !existingPolicy.getSecondaryWorkerConfig().equals(generatedPolicy.getSecondaryWorkerConfig());
            if (yarnDiff || workerDiff || secondaryWorkerDiff) {
                LOG.warn("The predefined auto-scaling policy {} already exists and is having a different configuration" + "as compared to CDF/CDAP's chosen configuration", existingPolicy.getName());
            }
        } catch (NotFoundException e) {
            createPolicy = true;
            LOG.debug("The autoscaling policy doesn't exists");
        }
        if (createPolicy) {
            RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
            try {
                autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, generatedPolicy);
            } catch (AlreadyExistsException e) {
            // no - op Race condition: in case 2 pipelines with auto-scaling are triggered together and might try to
            // create in parallel
            }
        }
    }
    return autoscalingPolicyName.toString();
}

Example 8 with AutoscalingPolicyServiceClient

use of com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient in project cdap by cdapio.

the class PredefinedAutoScalingTest method testFetchingExistingAutoScalingPolicySuccess.

@Test
public void testFetchingExistingAutoScalingPolicySuccess() throws IOException {
    PredefinedAutoScaling predefinedAutoScaling = new PredefinedAutoScaling(dataprocConf);
    // mock Return generated auto-scaling policy while fetching
    AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), PredefinedAutoScaling.AUTOSCALING_POLICY_ID);
    AutoscalingPolicy generatedPolicy = predefinedAutoScaling.generatePredefinedAutoScaling();
    AutoscalingPolicyServiceClient mockClient = PowerMockito.mock(AutoscalingPolicyServiceClient.class);
    Mockito.when(mockClient.getAutoscalingPolicy(autoscalingPolicyName)).thenReturn(generatedPolicy);
    PredefinedAutoScaling spy = Mockito.spy(predefinedAutoScaling);
    Mockito.doReturn(mockClient).when(spy).getAutoscalingPolicyServiceClient();
    String name = spy.createPredefinedAutoScalingPolicy();
    Mockito.verify(mockClient, Mockito.times(1)).getAutoscalingPolicy(autoscalingPolicyName);
    // verify that create call is not made
    RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
    Mockito.verify(mockClient, Mockito.never()).createAutoscalingPolicy(parent, generatedPolicy);
    Assert.assertEquals(name, autoscalingPolicyName.toString());
}

Example 9 with AutoscalingPolicyServiceClient

use of com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient in project cdap by cdapio.

the class PredefinedAutoScalingTest method testFetchFailedAndCreateIsCalled.

@Test
public void testFetchFailedAndCreateIsCalled() throws IOException {
    PredefinedAutoScaling predefinedAutoScaling = new PredefinedAutoScaling(dataprocConf);
    // mock Return generated auto-scaling policy while fetching
    AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), PredefinedAutoScaling.AUTOSCALING_POLICY_ID);
    AutoscalingPolicy generatedPolicy = predefinedAutoScaling.generatePredefinedAutoScaling();
    AutoscalingPolicyServiceClient mockClient = PowerMockito.mock(AutoscalingPolicyServiceClient.class);
    Mockito.when(mockClient.getAutoscalingPolicy(autoscalingPolicyName)).thenThrow(NotFoundException.class);
    RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
    Mockito.when(mockClient.createAutoscalingPolicy(parent, generatedPolicy)).thenReturn(null);
    PredefinedAutoScaling spy = Mockito.spy(predefinedAutoScaling);
    Mockito.doReturn(mockClient).when(spy).getAutoscalingPolicyServiceClient();
    String name = spy.createPredefinedAutoScalingPolicy();
    Mockito.verify(mockClient, Mockito.times(1)).getAutoscalingPolicy(autoscalingPolicyName);
    // verify that create call is not made
    Mockito.verify(mockClient, Mockito.times(1)).createAutoscalingPolicy(parent, generatedPolicy);
    Assert.assertEquals(name, autoscalingPolicyName.toString());
}

Example 10 with AutoscalingPolicyServiceClient

use of com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient in project java-dataproc by googleapis.

the class CreateClusterWithAutoscaling method createClusterwithAutoscaling.

public static void createClusterwithAutoscaling(String projectId, String region, String clusterName, String autoscalingPolicyName) throws IOException, InterruptedException {
    String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);
    // Configure the settings for the cluster controller client.
    ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
    // Configure the settings for the autoscaling policy service client.
    AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings = AutoscalingPolicyServiceSettings.newBuilder().setEndpoint(myEndpoint).build();
    // method.
    try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings);
        AutoscalingPolicyServiceClient autoscalingPolicyServiceClient = AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings)) {
        // Create the Autoscaling policy.
        InstanceGroupAutoscalingPolicyConfig workerInstanceGroupAutoscalingPolicyConfig = InstanceGroupAutoscalingPolicyConfig.newBuilder().setMinInstances(2).setMaxInstances(100).setWeight(1).build();
        InstanceGroupAutoscalingPolicyConfig secondaryWorkerInstanceGroupAutoscalingPolicyConfig = InstanceGroupAutoscalingPolicyConfig.newBuilder().setMinInstances(0).setMaxInstances(100).setWeight(1).build();
        BasicYarnAutoscalingConfig basicYarnApplicationConfig = BasicYarnAutoscalingConfig.newBuilder().setScaleUpFactor(0.05).setScaleDownFactor(1.0).setScaleUpMinWorkerFraction(0.0).setScaleUpMinWorkerFraction(0.0).setGracefulDecommissionTimeout(Duration.newBuilder().setSeconds(3600).build()).build();
        BasicAutoscalingAlgorithm basicAutoscalingAlgorithm = BasicAutoscalingAlgorithm.newBuilder().setCooldownPeriod(Duration.newBuilder().setSeconds(240).build()).setYarnConfig(basicYarnApplicationConfig).build();
        AutoscalingPolicy autoscalingPolicy = AutoscalingPolicy.newBuilder().setId(autoscalingPolicyName).setWorkerConfig(workerInstanceGroupAutoscalingPolicyConfig).setSecondaryWorkerConfig(secondaryWorkerInstanceGroupAutoscalingPolicyConfig).setBasicAlgorithm(basicAutoscalingAlgorithm).build();
        RegionName parent = RegionName.of(projectId, region);
        // Policy is uploaded here.
        autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, autoscalingPolicy);
        // Now the policy can be referenced when creating a cluster.
        String autoscalingPolicyUri = String.format("projects/%s/locations/%s/autoscalingPolicies/%s", projectId, region, autoscalingPolicyName);
        AutoscalingConfig autoscalingConfig = AutoscalingConfig.newBuilder().setPolicyUri(autoscalingPolicyUri).build();
        // Configure the settings for our cluster.
        InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder().setMachineTypeUri("n1-standard-2").setNumInstances(1).build();
        InstanceGroupConfig workerConfig = InstanceGroupConfig.newBuilder().setMachineTypeUri("n1-standard-2").setNumInstances(2).build();
        ClusterConfig clusterConfig = ClusterConfig.newBuilder().setMasterConfig(masterConfig).setWorkerConfig(workerConfig).setAutoscalingConfig(autoscalingConfig).build();
        // Create the cluster object with the desired cluster config.
        Cluster cluster = Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build();
        // Create the Dataproc cluster.
        OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest = clusterControllerClient.createClusterAsync(projectId, region, cluster);
        Cluster response = createClusterAsyncRequest.get();
        // Print out a success message.
        System.out.printf("Cluster created successfully: %s", response.getClusterName());
    } catch (ExecutionException e) {
        // If cluster creation does not complete successfully, print the error message.
        System.err.println(String.format("createClusterWithAutoscaling: %s ", e.getMessage()));
    }
}