Search in sources :

Example 1 with AutoscalingPolicyServiceSettings

use of com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings in project java-dataproc by googleapis.

the class CreateClusterWithAutoscalingTest method tearDown.

@After
public void tearDown() throws IOException, InterruptedException, ExecutionException {
    String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION);
    ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
    AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings = AutoscalingPolicyServiceSettings.newBuilder().setEndpoint(myEndpoint).build();
    try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings);
        AutoscalingPolicyServiceClient autoscalingPolicyServiceClient = AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings)) {
        OperationFuture<Empty, ClusterOperationMetadata> deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME);
        deleteClusterAsyncRequest.get();
        AutoscalingPolicyName name = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(PROJECT_ID, REGION, AUTOSCALING_POLICY_NAME);
        autoscalingPolicyServiceClient.deleteAutoscalingPolicy(name);
    }
}
Also used : Empty(com.google.protobuf.Empty) ClusterOperationMetadata(com.google.cloud.dataproc.v1.ClusterOperationMetadata) ClusterControllerClient(com.google.cloud.dataproc.v1.ClusterControllerClient) AutoscalingPolicyServiceClient(com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient) AutoscalingPolicyName(com.google.cloud.dataproc.v1.AutoscalingPolicyName) ClusterControllerSettings(com.google.cloud.dataproc.v1.ClusterControllerSettings) AutoscalingPolicyServiceSettings(com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings) After(org.junit.After)

Example 2 with AutoscalingPolicyServiceSettings

use of com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings in project cdap by cdapio.

the class PredefinedAutoScaling method getAutoscalingPolicyServiceClient.

/*
   * Using the input Google Credentials retrieve the Dataproc Autoscaling Service client
   */
@VisibleForTesting
AutoscalingPolicyServiceClient getAutoscalingPolicyServiceClient() throws IOException {
    CredentialsProvider credentialsProvider = FixedCredentialsProvider.create(dataprocConf.getDataprocCredentials());
    String rootUrl = Optional.ofNullable(dataprocConf.getRootUrl()).orElse(ClusterControllerSettings.getDefaultEndpoint());
    String regionalEndpoint = dataprocConf.getRegion() + "-" + rootUrl;
    AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings = AutoscalingPolicyServiceSettings.newBuilder().setCredentialsProvider(credentialsProvider).setEndpoint(regionalEndpoint).build();
    return AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings);
}
Also used : FixedCredentialsProvider(com.google.api.gax.core.FixedCredentialsProvider) CredentialsProvider(com.google.api.gax.core.CredentialsProvider) AutoscalingPolicyServiceSettings(com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 3 with AutoscalingPolicyServiceSettings

use of com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings in project cdap by caskdata.

the class PredefinedAutoScaling method getAutoscalingPolicyServiceClient.

/*
   * Using the input Google Credentials retrieve the Dataproc Autoscaling Service client
   */
@VisibleForTesting
AutoscalingPolicyServiceClient getAutoscalingPolicyServiceClient() throws IOException {
    CredentialsProvider credentialsProvider = FixedCredentialsProvider.create(dataprocConf.getDataprocCredentials());
    String rootUrl = Optional.ofNullable(dataprocConf.getRootUrl()).orElse(ClusterControllerSettings.getDefaultEndpoint());
    String regionalEndpoint = dataprocConf.getRegion() + "-" + rootUrl;
    AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings = AutoscalingPolicyServiceSettings.newBuilder().setCredentialsProvider(credentialsProvider).setEndpoint(regionalEndpoint).build();
    return AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings);
}
Also used : FixedCredentialsProvider(com.google.api.gax.core.FixedCredentialsProvider) CredentialsProvider(com.google.api.gax.core.CredentialsProvider) AutoscalingPolicyServiceSettings(com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 4 with AutoscalingPolicyServiceSettings

use of com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings in project java-dataproc by googleapis.

the class CreateClusterWithAutoscaling method createClusterwithAutoscaling.

public static void createClusterwithAutoscaling(String projectId, String region, String clusterName, String autoscalingPolicyName) throws IOException, InterruptedException {
    String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);
    // Configure the settings for the cluster controller client.
    ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
    // Configure the settings for the autoscaling policy service client.
    AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings = AutoscalingPolicyServiceSettings.newBuilder().setEndpoint(myEndpoint).build();
    // method.
    try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings);
        AutoscalingPolicyServiceClient autoscalingPolicyServiceClient = AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings)) {
        // Create the Autoscaling policy.
        InstanceGroupAutoscalingPolicyConfig workerInstanceGroupAutoscalingPolicyConfig = InstanceGroupAutoscalingPolicyConfig.newBuilder().setMinInstances(2).setMaxInstances(100).setWeight(1).build();
        InstanceGroupAutoscalingPolicyConfig secondaryWorkerInstanceGroupAutoscalingPolicyConfig = InstanceGroupAutoscalingPolicyConfig.newBuilder().setMinInstances(0).setMaxInstances(100).setWeight(1).build();
        BasicYarnAutoscalingConfig basicYarnApplicationConfig = BasicYarnAutoscalingConfig.newBuilder().setScaleUpFactor(0.05).setScaleDownFactor(1.0).setScaleUpMinWorkerFraction(0.0).setScaleUpMinWorkerFraction(0.0).setGracefulDecommissionTimeout(Duration.newBuilder().setSeconds(3600).build()).build();
        BasicAutoscalingAlgorithm basicAutoscalingAlgorithm = BasicAutoscalingAlgorithm.newBuilder().setCooldownPeriod(Duration.newBuilder().setSeconds(240).build()).setYarnConfig(basicYarnApplicationConfig).build();
        AutoscalingPolicy autoscalingPolicy = AutoscalingPolicy.newBuilder().setId(autoscalingPolicyName).setWorkerConfig(workerInstanceGroupAutoscalingPolicyConfig).setSecondaryWorkerConfig(secondaryWorkerInstanceGroupAutoscalingPolicyConfig).setBasicAlgorithm(basicAutoscalingAlgorithm).build();
        RegionName parent = RegionName.of(projectId, region);
        // Policy is uploaded here.
        autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, autoscalingPolicy);
        // Now the policy can be referenced when creating a cluster.
        String autoscalingPolicyUri = String.format("projects/%s/locations/%s/autoscalingPolicies/%s", projectId, region, autoscalingPolicyName);
        AutoscalingConfig autoscalingConfig = AutoscalingConfig.newBuilder().setPolicyUri(autoscalingPolicyUri).build();
        // Configure the settings for our cluster.
        InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder().setMachineTypeUri("n1-standard-2").setNumInstances(1).build();
        InstanceGroupConfig workerConfig = InstanceGroupConfig.newBuilder().setMachineTypeUri("n1-standard-2").setNumInstances(2).build();
        ClusterConfig clusterConfig = ClusterConfig.newBuilder().setMasterConfig(masterConfig).setWorkerConfig(workerConfig).setAutoscalingConfig(autoscalingConfig).build();
        // Create the cluster object with the desired cluster config.
        Cluster cluster = Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build();
        // Create the Dataproc cluster.
        OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest = clusterControllerClient.createClusterAsync(projectId, region, cluster);
        Cluster response = createClusterAsyncRequest.get();
        // Print out a success message.
        System.out.printf("Cluster created successfully: %s", response.getClusterName());
    } catch (ExecutionException e) {
        // If cluster creation does not complete successfully, print the error message.
        System.err.println(String.format("createClusterWithAutoscaling: %s ", e.getMessage()));
    }
}
Also used : ClusterOperationMetadata(com.google.cloud.dataproc.v1.ClusterOperationMetadata) BasicAutoscalingAlgorithm(com.google.cloud.dataproc.v1.BasicAutoscalingAlgorithm) Cluster(com.google.cloud.dataproc.v1.Cluster) AutoscalingPolicyServiceClient(com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient) ClusterControllerSettings(com.google.cloud.dataproc.v1.ClusterControllerSettings) AutoscalingPolicyServiceSettings(com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings) InstanceGroupAutoscalingPolicyConfig(com.google.cloud.dataproc.v1.InstanceGroupAutoscalingPolicyConfig) AutoscalingPolicy(com.google.cloud.dataproc.v1.AutoscalingPolicy) RegionName(com.google.cloud.dataproc.v1.RegionName) AutoscalingConfig(com.google.cloud.dataproc.v1.AutoscalingConfig) BasicYarnAutoscalingConfig(com.google.cloud.dataproc.v1.BasicYarnAutoscalingConfig) ClusterControllerClient(com.google.cloud.dataproc.v1.ClusterControllerClient) BasicYarnAutoscalingConfig(com.google.cloud.dataproc.v1.BasicYarnAutoscalingConfig) ExecutionException(java.util.concurrent.ExecutionException) InstanceGroupConfig(com.google.cloud.dataproc.v1.InstanceGroupConfig) ClusterConfig(com.google.cloud.dataproc.v1.ClusterConfig)

Aggregations

AutoscalingPolicyServiceSettings (com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings)4 CredentialsProvider (com.google.api.gax.core.CredentialsProvider)2 FixedCredentialsProvider (com.google.api.gax.core.FixedCredentialsProvider)2 AutoscalingPolicyServiceClient (com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient)2 ClusterControllerClient (com.google.cloud.dataproc.v1.ClusterControllerClient)2 ClusterControllerSettings (com.google.cloud.dataproc.v1.ClusterControllerSettings)2 ClusterOperationMetadata (com.google.cloud.dataproc.v1.ClusterOperationMetadata)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)2 AutoscalingConfig (com.google.cloud.dataproc.v1.AutoscalingConfig)1 AutoscalingPolicy (com.google.cloud.dataproc.v1.AutoscalingPolicy)1 AutoscalingPolicyName (com.google.cloud.dataproc.v1.AutoscalingPolicyName)1 BasicAutoscalingAlgorithm (com.google.cloud.dataproc.v1.BasicAutoscalingAlgorithm)1 BasicYarnAutoscalingConfig (com.google.cloud.dataproc.v1.BasicYarnAutoscalingConfig)1 Cluster (com.google.cloud.dataproc.v1.Cluster)1 ClusterConfig (com.google.cloud.dataproc.v1.ClusterConfig)1 InstanceGroupAutoscalingPolicyConfig (com.google.cloud.dataproc.v1.InstanceGroupAutoscalingPolicyConfig)1 InstanceGroupConfig (com.google.cloud.dataproc.v1.InstanceGroupConfig)1 RegionName (com.google.cloud.dataproc.v1.RegionName)1 Empty (com.google.protobuf.Empty)1 ExecutionException (java.util.concurrent.ExecutionException)1