use of com.google.cloud.dataproc.v1.AutoscalingPolicyName in project java-dataproc by googleapis.
the class CreateClusterWithAutoscalingTest method tearDown.
@After
public void tearDown() throws IOException, InterruptedException, ExecutionException {
String myEndpoint = String.format("%s-dataproc.googleapis.com:443", REGION);
ClusterControllerSettings clusterControllerSettings = ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings = AutoscalingPolicyServiceSettings.newBuilder().setEndpoint(myEndpoint).build();
try (ClusterControllerClient clusterControllerClient = ClusterControllerClient.create(clusterControllerSettings);
AutoscalingPolicyServiceClient autoscalingPolicyServiceClient = AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings)) {
OperationFuture<Empty, ClusterOperationMetadata> deleteClusterAsyncRequest = clusterControllerClient.deleteClusterAsync(PROJECT_ID, REGION, CLUSTER_NAME);
deleteClusterAsyncRequest.get();
AutoscalingPolicyName name = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(PROJECT_ID, REGION, AUTOSCALING_POLICY_NAME);
autoscalingPolicyServiceClient.deleteAutoscalingPolicy(name);
}
}
use of com.google.cloud.dataproc.v1.AutoscalingPolicyName in project cdap by caskdata.
the class PredefinedAutoScaling method createPredefinedAutoScalingPolicy.
// Creates the auto-scaling policy if doesn't exist.
public String createPredefinedAutoScalingPolicy() throws IOException {
AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), AUTOSCALING_POLICY_ID);
AutoscalingPolicy generatedPolicy = generatePredefinedAutoScaling();
try (AutoscalingPolicyServiceClient autoscalingPolicyServiceClient = getAutoscalingPolicyServiceClient()) {
// If the policy already exists meaning it is already created by cdap or in rare scenario the customer might have
// created with a similar name. It is also possible that the user might have changed configs in the autoscaling
// policy.
// So, if exists -> we will fetch and check if it differs from our predefined values and LOG it
AutoscalingPolicy existingPolicy = null;
boolean createPolicy = false;
try {
existingPolicy = autoscalingPolicyServiceClient.getAutoscalingPolicy(autoscalingPolicyName);
boolean yarnDiff = !existingPolicy.getBasicAlgorithm().getYarnConfig().equals(generatedPolicy.getBasicAlgorithm().getYarnConfig());
boolean workerDiff = !existingPolicy.getWorkerConfig().equals(generatedPolicy.getWorkerConfig());
boolean secondaryWorkerDiff = !existingPolicy.getSecondaryWorkerConfig().equals(generatedPolicy.getSecondaryWorkerConfig());
if (yarnDiff || workerDiff || secondaryWorkerDiff) {
LOG.warn("The predefined auto-scaling policy {} already exists and is having a different configuration" + "as compared to CDF/CDAP's chosen configuration", existingPolicy.getName());
}
} catch (NotFoundException e) {
createPolicy = true;
LOG.debug("The autoscaling policy doesn't exists");
}
if (createPolicy) {
RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
try {
autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, generatedPolicy);
} catch (AlreadyExistsException e) {
// no - op Race condition: in case 2 pipelines with auto-scaling are triggered together and might try to
// create in parallel
}
}
}
return autoscalingPolicyName.toString();
}
use of com.google.cloud.dataproc.v1.AutoscalingPolicyName in project cdap by caskdata.
the class PredefinedAutoScalingTest method testFetchFailedAndCreateIsCalled.
@Test
public void testFetchFailedAndCreateIsCalled() throws IOException {
PredefinedAutoScaling predefinedAutoScaling = new PredefinedAutoScaling(dataprocConf);
// mock Return generated auto-scaling policy while fetching
AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), PredefinedAutoScaling.AUTOSCALING_POLICY_ID);
AutoscalingPolicy generatedPolicy = predefinedAutoScaling.generatePredefinedAutoScaling();
AutoscalingPolicyServiceClient mockClient = PowerMockito.mock(AutoscalingPolicyServiceClient.class);
Mockito.when(mockClient.getAutoscalingPolicy(autoscalingPolicyName)).thenThrow(NotFoundException.class);
RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
Mockito.when(mockClient.createAutoscalingPolicy(parent, generatedPolicy)).thenReturn(null);
PredefinedAutoScaling spy = Mockito.spy(predefinedAutoScaling);
Mockito.doReturn(mockClient).when(spy).getAutoscalingPolicyServiceClient();
String name = spy.createPredefinedAutoScalingPolicy();
Mockito.verify(mockClient, Mockito.times(1)).getAutoscalingPolicy(autoscalingPolicyName);
// verify that create call is not made
Mockito.verify(mockClient, Mockito.times(1)).createAutoscalingPolicy(parent, generatedPolicy);
Assert.assertEquals(name, autoscalingPolicyName.toString());
}
use of com.google.cloud.dataproc.v1.AutoscalingPolicyName in project cdap by caskdata.
the class PredefinedAutoScalingTest method testFetchingExistingAutoScalingPolicySuccess.
@Test
public void testFetchingExistingAutoScalingPolicySuccess() throws IOException {
PredefinedAutoScaling predefinedAutoScaling = new PredefinedAutoScaling(dataprocConf);
// mock Return generated auto-scaling policy while fetching
AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), PredefinedAutoScaling.AUTOSCALING_POLICY_ID);
AutoscalingPolicy generatedPolicy = predefinedAutoScaling.generatePredefinedAutoScaling();
AutoscalingPolicyServiceClient mockClient = PowerMockito.mock(AutoscalingPolicyServiceClient.class);
Mockito.when(mockClient.getAutoscalingPolicy(autoscalingPolicyName)).thenReturn(generatedPolicy);
PredefinedAutoScaling spy = Mockito.spy(predefinedAutoScaling);
Mockito.doReturn(mockClient).when(spy).getAutoscalingPolicyServiceClient();
String name = spy.createPredefinedAutoScalingPolicy();
Mockito.verify(mockClient, Mockito.times(1)).getAutoscalingPolicy(autoscalingPolicyName);
// verify that create call is not made
RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
Mockito.verify(mockClient, Mockito.never()).createAutoscalingPolicy(parent, generatedPolicy);
Assert.assertEquals(name, autoscalingPolicyName.toString());
}
use of com.google.cloud.dataproc.v1.AutoscalingPolicyName in project cdap by cdapio.
the class PredefinedAutoScaling method createPredefinedAutoScalingPolicy.
// Creates the auto-scaling policy if doesn't exist.
public String createPredefinedAutoScalingPolicy() throws IOException {
AutoscalingPolicyName autoscalingPolicyName = AutoscalingPolicyName.ofProjectLocationAutoscalingPolicyName(dataprocConf.getProjectId(), dataprocConf.getRegion(), AUTOSCALING_POLICY_ID);
AutoscalingPolicy generatedPolicy = generatePredefinedAutoScaling();
try (AutoscalingPolicyServiceClient autoscalingPolicyServiceClient = getAutoscalingPolicyServiceClient()) {
// If the policy already exists meaning it is already created by cdap or in rare scenario the customer might have
// created with a similar name. It is also possible that the user might have changed configs in the autoscaling
// policy.
// So, if exists -> we will fetch and check if it differs from our predefined values and LOG it
AutoscalingPolicy existingPolicy = null;
boolean createPolicy = false;
try {
existingPolicy = autoscalingPolicyServiceClient.getAutoscalingPolicy(autoscalingPolicyName);
boolean yarnDiff = !existingPolicy.getBasicAlgorithm().getYarnConfig().equals(generatedPolicy.getBasicAlgorithm().getYarnConfig());
boolean workerDiff = !existingPolicy.getWorkerConfig().equals(generatedPolicy.getWorkerConfig());
boolean secondaryWorkerDiff = !existingPolicy.getSecondaryWorkerConfig().equals(generatedPolicy.getSecondaryWorkerConfig());
if (yarnDiff || workerDiff || secondaryWorkerDiff) {
LOG.warn("The predefined auto-scaling policy {} already exists and is having a different configuration" + "as compared to CDF/CDAP's chosen configuration", existingPolicy.getName());
}
} catch (NotFoundException e) {
createPolicy = true;
LOG.debug("The autoscaling policy doesn't exists");
}
if (createPolicy) {
RegionName parent = RegionName.of(dataprocConf.getProjectId(), dataprocConf.getRegion());
try {
autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, generatedPolicy);
} catch (AlreadyExistsException e) {
// no - op Race condition: in case 2 pipelines with auto-scaling are triggered together and might try to
// create in parallel
}
}
}
return autoscalingPolicyName.toString();
}
Aggregations