use of com.google.container.v1beta1.Cluster in project java-dataproc by googleapis.
the class CreateClusterWithAutoscaling method createClusterwithAutoscaling.
public static void createClusterwithAutoscaling(String projectId, String region, String clusterName, String autoscalingPolicyName) throws IOException, InterruptedException {
String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);
// Configure the settings for the cluster controller client.
ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
// Configure the settings for the autoscaling policy service client.
AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings = AutoscalingPolicyServiceSettings.newBuilder().setEndpoint(myEndpoint).build();
// method.
try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings);
AutoscalingPolicyServiceClient autoscalingPolicyServiceClient = AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings)) {
// Create the Autoscaling policy.
InstanceGroupAutoscalingPolicyConfig workerInstanceGroupAutoscalingPolicyConfig = InstanceGroupAutoscalingPolicyConfig.newBuilder().setMinInstances(2).setMaxInstances(100).setWeight(1).build();
InstanceGroupAutoscalingPolicyConfig secondaryWorkerInstanceGroupAutoscalingPolicyConfig = InstanceGroupAutoscalingPolicyConfig.newBuilder().setMinInstances(0).setMaxInstances(100).setWeight(1).build();
BasicYarnAutoscalingConfig basicYarnApplicationConfig = BasicYarnAutoscalingConfig.newBuilder().setScaleUpFactor(0.05).setScaleDownFactor(1.0).setScaleUpMinWorkerFraction(0.0).setScaleUpMinWorkerFraction(0.0).setGracefulDecommissionTimeout(Duration.newBuilder().setSeconds(3600).build()).build();
BasicAutoscalingAlgorithm basicAutoscalingAlgorithm = BasicAutoscalingAlgorithm.newBuilder().setCooldownPeriod(Duration.newBuilder().setSeconds(240).build()).setYarnConfig(basicYarnApplicationConfig).build();
AutoscalingPolicy autoscalingPolicy = AutoscalingPolicy.newBuilder().setId(autoscalingPolicyName).setWorkerConfig(workerInstanceGroupAutoscalingPolicyConfig).setSecondaryWorkerConfig(secondaryWorkerInstanceGroupAutoscalingPolicyConfig).setBasicAlgorithm(basicAutoscalingAlgorithm).build();
RegionName parent = RegionName.of(projectId, region);
// Policy is uploaded here.
autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, autoscalingPolicy);
// Now the policy can be referenced when creating a cluster.
String autoscalingPolicyUri = String.format("projects/%s/locations/%s/autoscalingPolicies/%s", projectId, region, autoscalingPolicyName);
AutoscalingConfig autoscalingConfig = AutoscalingConfig.newBuilder().setPolicyUri(autoscalingPolicyUri).build();
// Configure the settings for our cluster.
InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder().setMachineTypeUri("n1-standard-2").setNumInstances(1).build();
InstanceGroupConfig workerConfig = InstanceGroupConfig.newBuilder().setMachineTypeUri("n1-standard-2").setNumInstances(2).build();
ClusterConfig clusterConfig = ClusterConfig.newBuilder().setMasterConfig(masterConfig).setWorkerConfig(workerConfig).setAutoscalingConfig(autoscalingConfig).build();
// Create the cluster object with the desired cluster config.
Cluster cluster = Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build();
// Create the Dataproc cluster.
OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest = clusterControllerClient.createClusterAsync(projectId, region, cluster);
Cluster response = createClusterAsyncRequest.get();
// Print out a success message.
System.out.printf("Cluster created successfully: %s", response.getClusterName());
} catch (ExecutionException e) {
// If cluster creation does not complete successfully, print the error message.
System.err.println(String.format("createClusterWithAutoscaling: %s ", e.getMessage()));
}
}
use of com.google.container.v1beta1.Cluster in project java-dataproc by googleapis.
the class Quickstart method quickstart.
public static void quickstart(String projectId, String region, String clusterName, String jobFilePath) throws IOException, InterruptedException {
String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);
// Configure the settings for the cluster controller client.
ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
// Configure the settings for the job controller client.
JobControllerSettings jobControllerSettings = JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
// manually with the .close() method.
try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings);
JobControllerClient jobControllerClient = JobControllerClient.create(jobControllerSettings)) {
// Configure the settings for our cluster.
InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder().setMachineTypeUri("n1-standard-2").setNumInstances(1).build();
InstanceGroupConfig workerConfig = InstanceGroupConfig.newBuilder().setMachineTypeUri("n1-standard-2").setNumInstances(2).build();
ClusterConfig clusterConfig = ClusterConfig.newBuilder().setMasterConfig(masterConfig).setWorkerConfig(workerConfig).build();
// Create the cluster object with the desired cluster config.
Cluster cluster = Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build();
// Create the Cloud Dataproc cluster.
OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest = clusterControllerClient.createClusterAsync(projectId, region, cluster);
Cluster clusterResponse = createClusterAsyncRequest.get();
System.out.println(String.format("Cluster created successfully: %s", clusterResponse.getClusterName()));
// Configure the settings for our job.
JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build();
PySparkJob pySparkJob = PySparkJob.newBuilder().setMainPythonFileUri(jobFilePath).build();
Job job = Job.newBuilder().setPlacement(jobPlacement).setPysparkJob(pySparkJob).build();
// Submit an asynchronous request to execute the job.
OperationFuture<Job, JobMetadata> submitJobAsOperationAsyncRequest = jobControllerClient.submitJobAsOperationAsync(projectId, region, job);
Job jobResponse = submitJobAsOperationAsyncRequest.get();
// Print output from Google Cloud Storage.
Matcher matches = Pattern.compile("gs://(.*?)/(.*)").matcher(jobResponse.getDriverOutputResourceUri());
matches.matches();
Storage storage = StorageOptions.getDefaultInstance().getService();
Blob blob = storage.get(matches.group(1), String.format("%s.000000000", matches.group(2)));
System.out.println(String.format("Job finished successfully: %s", new String(blob.getContent())));
// Delete the cluster.
OperationFuture<Empty, ClusterOperationMetadata> deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(projectId, region, clusterName);
deleteClusterAsyncRequest.get();
System.out.println(String.format("Cluster \"%s\" successfully deleted.", clusterName));
} catch (ExecutionException e) {
System.err.println(String.format("quickstart: %s ", e.getMessage()));
}
}
use of com.google.container.v1beta1.Cluster in project java-dataproc by googleapis.
the class SubmitHadoopFsJobTest method setUp.
@Before
public void setUp() throws IOException, ExecutionException, InterruptedException {
bout = new ByteArrayOutputStream();
System.setOut(new PrintStream(bout));
ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build();
try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) {
// Create the Dataproc cluster.
Cluster cluster = Cluster.newBuilder().setClusterName(CLUSTER_NAME).build();
OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest = clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster);
createClusterAsyncRequest.get();
}
}
use of com.google.container.v1beta1.Cluster in project java-dataproc by googleapis.
the class SubmitJobTest method setUp.
@Before
public void setUp() throws IOException, ExecutionException, InterruptedException {
bout = new ByteArrayOutputStream();
System.setOut(new PrintStream(bout));
ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(ENDPOINT).build();
try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) {
// Create the Dataproc cluster.
Cluster cluster = Cluster.newBuilder().setClusterName(CLUSTER_NAME).build();
OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest = clusterControllerClient.createClusterAsync(PROJECT_ID, REGION, cluster);
createClusterAsyncRequest.get();
}
}
use of com.google.container.v1beta1.Cluster in project java-pubsublite-spark by googleapis.
the class AdminUtils method createCluster.
public static void createCluster(String projectId, String region, String clusterName, String imageVersion) throws IOException, InterruptedException {
String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);
// Configure the settings for the cluster controller client.
ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
// closes the client, but this can also be done manually with the .close() method.
try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings)) {
// Configure the settings for our cluster.
InstanceGroupConfig masterConfig = InstanceGroupConfig.newBuilder().setMachineTypeUri("n1-standard-2").setNumInstances(1).build();
InstanceGroupConfig workerConfig = InstanceGroupConfig.newBuilder().setMachineTypeUri("n1-standard-2").setNumInstances(2).build();
SoftwareConfig softwareConfig = SoftwareConfig.newBuilder().setImageVersion(imageVersion).build();
GceClusterConfig gceClusterConfig = GceClusterConfig.newBuilder().addServiceAccountScopes("https://www.googleapis.com/auth/cloud-platform").build();
// Delete the cluster after 2 hours without submitted jobs
LifecycleConfig lifecycleConfig = LifecycleConfig.newBuilder().setIdleDeleteTtl(Durations.fromHours(2L)).build();
ClusterConfig clusterConfig = ClusterConfig.newBuilder().setMasterConfig(masterConfig).setWorkerConfig(workerConfig).setSoftwareConfig(softwareConfig).setGceClusterConfig(gceClusterConfig).setLifecycleConfig(lifecycleConfig).build();
// Create the cluster object with the desired cluster config.
Cluster cluster = Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build();
// Create the Cloud Dataproc cluster.
OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest = clusterControllerClient.createClusterAsync(projectId, region, cluster);
Cluster response = createClusterAsyncRequest.get();
// Print out a success message.
System.out.printf("Cluster created successfully: %s", response.getClusterName());
} catch (ExecutionException e) {
System.err.println(String.format("Error executing createCluster: %s ", e.getMessage()));
}
}
Aggregations