Examples with CloudbreakNode com.sequenceiq.cloudbreak.domain.CloudbreakNode used on opensource projects

Example 6 with CloudbreakNode

use of com.sequenceiq.cloudbreak.domain.CloudbreakNode in project cloudbreak by hortonworks.

the class HeartbeatServiceTest method testOneNodeTakesAllFlowsWithCleanup.

@Test
public void testOneNodeTakesAllFlowsWithCleanup() {
    List<CloudbreakNode> clusterNodes = getClusterNodes();
    // myself
    clusterNodes.get(0).setLastUpdated(200_000);
    // set all nodes to failed except myself
    for (int i = 1; i < clusterNodes.size(); i++) {
        CloudbreakNode node = clusterNodes.get(i);
        node.setLastUpdated(50_000);
    }
    when(cloudbreakNodeRepository.findAll()).thenReturn(clusterNodes);
    when(clock.getCurrentTime()).thenReturn(200_000L);
    // all flows that need to be re-distributed
    List<String> suspendedFlows = new ArrayList<>();
    List<FlowLog> node1FlowLogs = getFlowLogs(2, 5000);
    suspendedFlows.addAll(node1FlowLogs.stream().map(FlowLog::getFlowId).distinct().collect(Collectors.toList()));
    when(flowLogRepository.findAllByCloudbreakNodeId(NODE_1_ID)).thenReturn(new HashSet<>(node1FlowLogs)).thenReturn(Collections.emptySet());
    Set<FlowLog> node2FlowLogs = new HashSet<>(getFlowLogs(3, 3000));
    suspendedFlows.addAll(node2FlowLogs.stream().map(FlowLog::getFlowId).distinct().collect(Collectors.toList()));
    when(flowLogRepository.findAllByCloudbreakNodeId(NODE_2_ID)).thenReturn(node2FlowLogs).thenReturn(Collections.emptySet());
    Map<CloudbreakNode, List<String>> distribution = new HashMap<>();
    distribution.computeIfAbsent(clusterNodes.get(0), v -> new ArrayList<>()).addAll(Arrays.asList(suspendedFlows.get(0), suspendedFlows.get(1), suspendedFlows.get(2), suspendedFlows.get(3), suspendedFlows.get(4)));
    when(flowDistributor.distribute(any(), any())).thenReturn(distribution);
    Set<FlowLog> myNewFlowLogs = new HashSet<>();
    myNewFlowLogs.addAll(node1FlowLogs);
    myNewFlowLogs.addAll(node2FlowLogs);
    when(flowLogRepository.findAllByCloudbreakNodeId(MY_ID)).thenReturn(myNewFlowLogs);
    when(runningFlows.get(any())).thenReturn(null);
    heartbeatService.scheduledFlowDistribution();
    verify(flowLogRepository).save(flowLogListCaptor.capture());
    List<FlowLog> updatedFlows = flowLogListCaptor.getValue();
    assertEquals(myNewFlowLogs.size(), updatedFlows.size());
    for (FlowLog updatedFlow : updatedFlows) {
        assertEquals(MY_ID, updatedFlow.getCloudbreakNodeId());
    }
    verify(flow2Handler, times(5)).restartFlow(stringCaptor.capture());
    List<String> allFlowIds = stringCaptor.getAllValues();
    assertEquals(5, allFlowIds.size());
    for (String flowId : suspendedFlows) {
        assertTrue(allFlowIds.contains(flowId));
    }
}

Example 7 with CloudbreakNode

use of com.sequenceiq.cloudbreak.domain.CloudbreakNode in project cloudbreak by hortonworks.

the class HeartbeatServiceTest method testDistributionConcurrencyWithDifferentFlows.

@Test
public void testDistributionConcurrencyWithDifferentFlows() {
    List<CloudbreakNode> clusterNodes = getClusterNodes();
    // myself
    clusterNodes.get(0).setLastUpdated(200_000);
    // failed node
    clusterNodes.get(1).setLastUpdated(50_000);
    // active node
    clusterNodes.get(2).setLastUpdated(200_000);
    when(cloudbreakNodeRepository.findAll()).thenReturn(clusterNodes);
    when(clock.getCurrentTime()).thenReturn(200_000L);
    // all flows that need to be re-distributed
    List<String> suspendedFlows = new ArrayList<>();
    List<FlowLog> node1FlowLogs = getFlowLogs(3, 5000);
    suspendedFlows.addAll(node1FlowLogs.stream().map(FlowLog::getFlowId).distinct().collect(Collectors.toList()));
    when(flowLogRepository.findAllByCloudbreakNodeId(NODE_1_ID)).thenReturn(new HashSet<>(node1FlowLogs));
    Map<CloudbreakNode, List<String>> distribution = new HashMap<>();
    distribution.computeIfAbsent(clusterNodes.get(0), v -> new ArrayList<>()).addAll(Arrays.asList(suspendedFlows.get(0), suspendedFlows.get(2)));
    distribution.computeIfAbsent(clusterNodes.get(2), v -> new ArrayList<>()).addAll(Collections.singletonList(suspendedFlows.get(1)));
    when(flowDistributor.distribute(any(), any())).thenReturn(distribution);
    List<FlowLog> myNewFlowLogs = node1FlowLogs.stream().filter(fl -> fl.getFlowId().equalsIgnoreCase(suspendedFlows.get(0))).collect(Collectors.toList());
    when(flowLogRepository.findAllByCloudbreakNodeId(MY_ID)).thenReturn(new HashSet<>(myNewFlowLogs));
    when(runningFlows.get(any())).thenReturn(null);
    when(flowLogRepository.save(anyCollection())).thenThrow(new OptimisticLockingFailureException("Someone already distributed the flows.."));
    heartbeatService.scheduledFlowDistribution();
    verify(flow2Handler, times(1)).restartFlow(stringCaptor.capture());
    List<String> allFlowIds = stringCaptor.getAllValues();
    assertEquals(1, allFlowIds.size());
    for (FlowLog flowLog : myNewFlowLogs) {
        assertTrue(allFlowIds.contains(flowLog.getFlowId()));
    }
}

Example 8 with CloudbreakNode

use of com.sequenceiq.cloudbreak.domain.CloudbreakNode in project cloudbreak by hortonworks.

the class EvenFlowDistributorTest method testFlowDistributionSingleNode.

@Test
public void testFlowDistributionSingleNode() {
    CloudbreakNode node = new CloudbreakNode(MY_ID);
    List<String> flowLogs = getFlowIds(11);
    Map<CloudbreakNode, List<String>> result = flowDistributor.distribute(flowLogs, Collections.singletonList(node));
    assertEquals(11, result.get(node).size());
}

Example 9 with CloudbreakNode

use of com.sequenceiq.cloudbreak.domain.CloudbreakNode in project cloudbreak by hortonworks.

the class EvenFlowDistributorTest method getClusterNodes.

private List<CloudbreakNode> getClusterNodes() {
    List<CloudbreakNode> nodes = new ArrayList<>();
    nodes.add(new CloudbreakNode(MY_ID));
    nodes.add(new CloudbreakNode(NODE_1_ID));
    nodes.add(new CloudbreakNode(NODE_2_ID));
    nodes.add(new CloudbreakNode(NODE_3_ID));
    return nodes;
}

Example 10 with CloudbreakNode

use of com.sequenceiq.cloudbreak.domain.CloudbreakNode in project cloudbreak by hortonworks.

the class HeartbeatService method heartbeat.

@Scheduled(cron = "${cb.ha.heartbeat.rate:0/30 * * * * *}")
public void heartbeat() {
    if (cloudbreakNodeConfig.isNodeIdSpecified()) {
        String nodeId = cloudbreakNodeConfig.getId();
        try {
            retryService.testWith2SecDelayMax5Times(() -> {
                try {
                    CloudbreakNode self = cloudbreakNodeRepository.findOne(nodeId);
                    if (self == null) {
                        self = new CloudbreakNode(nodeId);
                    }
                    self.setLastUpdated(clock.getCurrentTime());
                    cloudbreakNodeRepository.save(self);
                    return true;
                } catch (RuntimeException e) {
                    LOGGER.error("Failed to update the heartbeat timestamp", e);
                    metricService.incrementMetricCounter(MetricType.HEARTBEAT_UPDATE_FAILED);
                    throw new ActionWentFailException(e.getMessage());
                }
            });
        } catch (ActionWentFailException af) {
            LOGGER.error(String.format("Failed to update the heartbeat timestamp 5 times for node %s: %s", nodeId, af.getMessage()));
            cancelEveryFlowWithoutDbUpdate();
        }
        cancelInvalidFlows();
    }
}