Examples with YarnScalingServiceV1Response com.sequenceiq.periscope.model.yarn.YarnScalingServiceV1Response used on opensource projects

Example 1 with YarnScalingServiceV1Response

use of com.sequenceiq.periscope.model.yarn.YarnScalingServiceV1Response in project cloudbreak by hortonworks.

the class CronTimeEvaluatorTest method getMockYarnScalingResponse.

private YarnScalingServiceV1Response getMockYarnScalingResponse(String instanceType, int yarnGivenDecommissionCount) {
    YarnScalingServiceV1Response yarnScalingReponse = new YarnScalingServiceV1Response();
    List decommissionCandidates = new ArrayList();
    for (int i = 1; i <= yarnGivenDecommissionCount; i++) {
        YarnScalingServiceV1Response.DecommissionCandidate decommissionCandidate = new YarnScalingServiceV1Response.DecommissionCandidate();
        decommissionCandidate.setAmCount(2);
        decommissionCandidate.setNodeId(fqdnBase + testHostGroup + i + ":8042");
        decommissionCandidates.add(decommissionCandidate);
    }
    yarnScalingReponse.setDecommissionCandidates(Map.of("candidates", decommissionCandidates));
    return yarnScalingReponse;
}

Example 2 with YarnScalingServiceV1Response

use of com.sequenceiq.periscope.model.yarn.YarnScalingServiceV1Response in project cloudbreak by hortonworks.

the class CronTimeEvaluatorTest method validateScheduleBasedScaling.

private ScalingEvent validateScheduleBasedScaling(String testMode, Integer currentHostGroupCount, Integer desiredNodeCount, Optional<Integer> yarnGivenDecommissionCount) throws Exception {
    TimeAlert alert = getAAlert(desiredNodeCount);
    StackV4Response stackV4Response = MockStackResponseGenerator.getMockStackV4Response(clusterCrn, testHostGroup, "testFqdn" + testHostGroup, currentHostGroupCount, 0);
    when(cloudbreakCommunicator.getByCrn(anyString())).thenReturn(stackV4Response);
    when(stackResponseUtils.getNodeCountForHostGroup(stackV4Response, testHostGroup)).thenCallRealMethod();
    when(scalingPolicyTargetCalculator.getDesiredAbsoluteNodeCount(any(ScalingEvent.class), anyInt())).thenCallRealMethod();
    when(dateService.isTrigger(any(TimeAlert.class), anyLong())).thenReturn(true);
    if (!"SCALE_UP_MODE".equals(testMode)) {
        YarnScalingServiceV1Response yarnScalingServiceV1Response = getMockYarnScalingResponse("test", yarnGivenDecommissionCount.get());
        when(stackResponseUtils.getCloudInstanceIdsForHostGroup(any(), any())).thenCallRealMethod();
        when(yarnMetricsClient.getYarnMetricsForCluster(any(Cluster.class), any(StackV4Response.class), anyString(), any(), any(Optional.class))).thenReturn(yarnScalingServiceV1Response);
        when(yarnResponseUtils.getYarnRecommendedDecommissionHostsForHostGroup(anyString(), any(YarnScalingServiceV1Response.class), any(Map.class), anyInt(), any(Optional.class), anyInt())).thenCallRealMethod();
    }
    underTest.publishIfNeeded(List.of(alert));
    VerificationMode verificationMode = "SCALE_UP_MODE".equals(testMode) ? never() : times(1);
    ArgumentCaptor<ScalingEvent> captor = ArgumentCaptor.forClass(ScalingEvent.class);
    verify(eventPublisher).publishEvent(captor.capture());
    verify(stackResponseUtils, verificationMode).getCloudInstanceIdsForHostGroup(any(), any());
    verify(yarnMetricsClient, verificationMode).getYarnMetricsForCluster(any(Cluster.class), any(StackV4Response.class), anyString(), anyString(), any(Optional.class));
    verify(yarnResponseUtils, verificationMode).getYarnRecommendedDecommissionHostsForHostGroup(anyString(), any(YarnScalingServiceV1Response.class), any(Map.class), anyInt(), any(Optional.class), anyInt());
    return captor.getValue();
}

Example 3 with YarnScalingServiceV1Response

use of com.sequenceiq.periscope.model.yarn.YarnScalingServiceV1Response in project cloudbreak by hortonworks.

the class YarnLoadEvaluatorTest method getMockYarnScalingResponse.

private YarnScalingServiceV1Response getMockYarnScalingResponse(String instanceType, int upScaleCount, int downScaleCount) {
    NewNodeManagerCandidates.Candidate candidate = new NewNodeManagerCandidates.Candidate();
    candidate.setCount(upScaleCount);
    candidate.setModelName(instanceType);
    NewNodeManagerCandidates candidates = new NewNodeManagerCandidates();
    candidates.setCandidates(List.of(candidate));
    YarnScalingServiceV1Response yarnScalingReponse = new YarnScalingServiceV1Response();
    if (upScaleCount > 0) {
        yarnScalingReponse.setNewNMCandidates(candidates);
    }
    List decommissionCandidates = new ArrayList();
    for (int i = 1; i <= downScaleCount; i++) {
        DecommissionCandidate decommissionCandidate = new DecommissionCandidate();
        decommissionCandidate.setAmCount(2);
        decommissionCandidate.setNodeId(fqdnBase + i + ":8042");
        decommissionCandidates.add(decommissionCandidate);
    }
    yarnScalingReponse.setDecommissionCandidates(Map.of("candidates", decommissionCandidates));
    return yarnScalingReponse;
}

Example 4 with YarnScalingServiceV1Response

use of com.sequenceiq.periscope.model.yarn.YarnScalingServiceV1Response in project cloudbreak by hortonworks.

the class YarnMetricsClient method getYarnMetricsForCluster.

@Retryable(value = Exception.class, maxAttempts = 2, backoff = @Backoff(delay = 5000))
public YarnScalingServiceV1Response getYarnMetricsForCluster(Cluster cluster, StackV4Response stackV4Response, String hostGroup, String pollingUserCrn, Optional<Integer> mandatoryDownScaleCount) throws Exception {
    TlsConfiguration tlsConfig = tlsSecurityService.getTls(cluster.getId());
    String clusterProxyUrl = clusterProxyConfigurationService.getClusterProxyUrl().orElseThrow(() -> new RuntimeException(String.format("ClusterProxy Not Configured for Cluster %s, " + " cannot query YARN Metrics.", cluster.getStackCrn())));
    Client restClient = RestClientUtil.createClient(tlsConfig.getServerCert(), tlsConfig.getClientCert(), tlsConfig.getClientKey(), true);
    String yarnApiUrl = String.format(YARN_API_URL, clusterProxyUrl, cluster.getStackCrn());
    InstanceConfig instanceConfig = yarnServiceConfigClient.getInstanceConfigFromCM(cluster, stackV4Response, hostGroup);
    YarnScalingServiceV1Request yarnScalingServiceV1Request = new YarnScalingServiceV1Request();
    yarnScalingServiceV1Request.setInstanceTypes(List.of(new HostGroupInstanceType(instanceConfig.getInstanceName(), instanceConfig.getMemoryInMb().intValue(), instanceConfig.getCoreCPU())));
    LOGGER.debug("Using actorCrn '{}' for Cluster '{}' yarn polling.", pollingUserCrn, cluster.getStackCrn());
    UriBuilder yarnMetricsURI = UriBuilder.fromPath(yarnApiUrl).queryParam(PARAM_UPSCALE_FACTOR_NODE_RESOURCE_TYPE, DEFAULT_UPSCALE_RESOURCE_TYPE);
    mandatoryDownScaleCount.ifPresent(scaleDownCount -> yarnMetricsURI.queryParam(PARAM_DOWNSCALE_FACTOR_IN_NODE_COUNT, stackV4Response.getNodeCount()));
    YarnScalingServiceV1Response yarnResponse = requestLogging.logResponseTime(() -> restClient.target(yarnMetricsURI).request().accept(MediaType.APPLICATION_JSON_VALUE).header(HEADER_ACTOR_CRN, pollingUserCrn).post(Entity.json(yarnScalingServiceV1Request), YarnScalingServiceV1Response.class), String.format("YarnScalingAPI query for cluster crn '%s'", cluster.getStackCrn()));
    LOGGER.info("YarnScalingAPI response for cluster crn '{}',  response '{}'", cluster.getStackCrn(), yarnResponse);
    return yarnResponse;
}

Example 5 with YarnScalingServiceV1Response

use of com.sequenceiq.periscope.model.yarn.YarnScalingServiceV1Response in project cloudbreak by hortonworks.

the class YarnLoadEvaluatorTest method captureScalingEvent.

private Optional<ScalingEvent> captureScalingEvent(int runningNodeHostGroupCount, int stoppedNodeHostGroupCount, int yarnUpscaleCount, int yarnDownscaleCount, boolean scalingEventExpected) throws Exception {
    MockitoAnnotations.openMocks(this);
    Cluster cluster = getARunningCluster();
    cluster.setStopStartScalingEnabled(true);
    String hostGroup = "compute";
    StackV4Response stackV4Response = MockStackResponseGenerator.getMockStackV4ResponseWithStoppedAndRunningNodes(CLOUDBREAK_STACK_CRN, hostGroup, fqdnBase, runningNodeHostGroupCount, stoppedNodeHostGroupCount);
    YarnScalingServiceV1Response upScale = getMockYarnScalingResponse(hostGroup, yarnUpscaleCount, yarnDownscaleCount);
    setupMocks(cluster, upScale, stackV4Response);
    underTest.setContext(new ClusterIdEvaluatorContext(AUTOSCALE_CLUSTER_ID));
    underTest.execute();
    Optional<ScalingEvent> scalingEventCaptured = Optional.empty();
    if (scalingEventExpected) {
        ArgumentCaptor<ScalingEvent> captor = ArgumentCaptor.forClass(ScalingEvent.class);
        verify(eventPublisher).publishEvent(captor.capture());
        scalingEventCaptured = Optional.of(captor.getValue());
    }
    return scalingEventCaptured;
}