Examples with FlowLog com.sequenceiq.flow.domain.FlowLog used on opensource projects

Example 56 with FlowLog

use of com.sequenceiq.flow.domain.FlowLog in project cloudbreak by hortonworks.

the class Flow2Handler method restartFlow.

public void restartFlow(String flowId) {
    FlowLog flowLog = flowLogService.findFirstByFlowIdOrderByCreatedDesc(flowId).orElseThrow(() -> new FlowNotFoundException(flowId));
    restartFlow(flowLog);
}

Example 57 with FlowLog

use of com.sequenceiq.flow.domain.FlowLog in project cloudbreak by hortonworks.

the class Flow2Handler method restartFlow.

public void restartFlow(FlowLog flowLog) {
    if (notSupportedFlowType(flowLog)) {
        try {
            LOGGER.error("Flow type or payload is not present on the classpath anymore. Terminating the flow {}.", flowLog);
            flowLogService.terminate(flowLog.getResourceId(), flowLog.getFlowId());
            return;
        } catch (TransactionExecutionException e) {
            throw new TransactionRuntimeExecutionException(e);
        }
    }
    if (flowLog.getFlowType() != null) {
        Optional<FlowConfiguration<?>> flowConfig = flowConfigs.stream().filter(fc -> flowLog.isFlowType(fc.getClass())).findFirst();
        try {
            String flowChainType = flowChainLogService.getFlowChainType(flowLog.getFlowChainId());
            Payload payload = (Payload) JsonReader.jsonToJava(flowLog.getPayload());
            Flow flow = flowConfig.get().createFlow(flowLog.getFlowId(), flowLog.getFlowChainId(), payload.getResourceId(), flowChainType);
            runningFlows.put(flow, flowLog.getFlowChainId());
            flowStatCache.put(flow.getFlowId(), flowLog.getFlowChainId(), payload.getResourceId(), flowConfig.get().getFlowOperationType().name(), flow.getFlowConfigClass(), true);
            if (flowLog.getFlowChainId() != null) {
                flowChainHandler.restoreFlowChain(flowLog.getFlowChainId());
            }
            Map<Object, Object> variables = (Map<Object, Object>) JsonReader.jsonToJava(flowLog.getVariables());
            flow.initialize(flowLog.getCurrentState(), variables);
            RestartAction restartAction = flowConfig.get().getRestartAction(flowLog.getNextEvent());
            if (restartAction != null) {
                LOGGER.debug("Restarting flow with id: '{}', flow chain id: '{}', flow type: '{}', restart action: '{}'", flow.getFlowId(), flowLog.getFlowChainId(), flowLog.getFlowType().getClassValue().getSimpleName(), restartAction.getClass().getSimpleName());
                Span span = tracer.buildSpan(flowLog.getCurrentState()).ignoreActiveSpan().start();
                restartAction.restart(new FlowParameters(flowLog.getFlowId(), flowLog.getFlowTriggerUserCrn(), flowLog.getOperationType().name(), span.context()), flowLog.getFlowChainId(), flowLog.getNextEvent(), payload);
                return;
            }
        } catch (RuntimeException e) {
            String message = String.format("Flow could not be restarted with id: '%s', flow chain id: '%s' and flow type: '%s'", flowLog.getFlowId(), flowLog.getFlowChainId(), flowLog.getFlowType().getClassValue().getSimpleName());
            LOGGER.error(message, e);
        }
        try {
            flowLogService.terminate(flowLog.getResourceId(), flowLog.getFlowId());
        } catch (TransactionExecutionException e) {
            throw new TransactionRuntimeExecutionException(e);
        }
    }
}

Example 58 with FlowLog

use of com.sequenceiq.flow.domain.FlowLog in project cloudbreak by hortonworks.

the class Flow2Handler method updateFlowLogStatusInTransaction.

private void updateFlowLogStatusInTransaction(String key, Payload payload, FlowParameters flowParameters, String flowChainId, Flow flow, MutableBoolean flowCancelled) throws TransactionExecutionException {
    transactionService.required(() -> {
        Optional<FlowLog> lastFlowLog = flowLogService.getLastFlowLog(flow.getFlowId());
        if (lastFlowLog.isPresent()) {
            String nodeId = nodeConfig.getId();
            FlowLog flowLog = lastFlowLog.get();
            if (flowLog.getFinalized() || flowLog.getCloudbreakNodeId() == null || flowLog.getCloudbreakNodeId().equals(nodeId)) {
                updateFlowLogStatus(key, payload, flowChainId, flow, flowLog, flowParameters);
            } else {
                LOGGER.info("Flow {} was handled by another node {}, current node ID is {}, abandoning.", flow.getFlowId(), flowLog.getCloudbreakNodeId(), nodeId);
                inMemoryCleanup.cancelFlowWithoutDbUpdate(flow.getFlowId());
                flowCancelled.setTrue();
            }
        } else {
            LOGGER.debug("Cannot find LastFlowLog with flowId: {}", flow.getFlowId());
        }
    });
}

Example 59 with FlowLog

use of com.sequenceiq.flow.domain.FlowLog in project cloudbreak by hortonworks.

the class HeartbeatServiceTest method testDistributionConcurrencyWithDifferentFlows.

@Test
public void testDistributionConcurrencyWithDifferentFlows() {
    List<Node> clusterNodes = getClusterNodes();
    // myself
    clusterNodes.get(0).setLastUpdated(200_000L);
    // failed node
    clusterNodes.get(1).setLastUpdated(50_000L);
    // active node
    clusterNodes.get(2).setLastUpdated(200_000L);
    when(nodeService.findAll()).thenReturn(clusterNodes);
    when(clock.getCurrentTimeMillis()).thenReturn(200_000L);
    // all flows that need to be re-distributed
    List<FlowLog> node1FlowLogs = getFlowLogs(3, 5000);
    List<String> suspendedFlows = node1FlowLogs.stream().map(FlowLog::getFlowId).distinct().collect(Collectors.toList());
    when(flowLogService.findAllByCloudbreakNodeId(NODE_1_ID)).thenReturn(new HashSet<>(node1FlowLogs));
    Map<Node, List<String>> distribution = new HashMap<>();
    distribution.computeIfAbsent(clusterNodes.get(0), v -> new ArrayList<>()).addAll(Arrays.asList(suspendedFlows.get(0), suspendedFlows.get(2)));
    distribution.computeIfAbsent(clusterNodes.get(2), v -> new ArrayList<>()).addAll(Collections.singletonList(suspendedFlows.get(1)));
    when(flowDistributor.distribute(any(), any())).thenReturn(distribution);
    List<FlowLog> myNewFlowLogs = node1FlowLogs.stream().filter(fl -> fl.getFlowId().equalsIgnoreCase(suspendedFlows.get(0))).collect(Collectors.toList());
    when(flowLogService.findAllByCloudbreakNodeId(MY_ID)).thenReturn(new HashSet<>(myNewFlowLogs));
    when(runningFlows.get(any())).thenReturn(null);
    when(flowLogService.saveAll(anyCollection())).thenThrow(new OptimisticLockingFailureException("Someone already distributed the flows.."));
    heartbeatService.scheduledFlowDistribution();
    verify(flow2Handler, times(1)).restartFlow(stringCaptor.capture());
    List<String> allFlowIds = stringCaptor.getAllValues();
    assertEquals(1L, allFlowIds.size());
    for (FlowLog flowLog : myNewFlowLogs) {
        assertTrue(allFlowIds.contains(flowLog.getFlowId()));
    }
}

Example 60 with FlowLog

use of com.sequenceiq.flow.domain.FlowLog in project cloudbreak by hortonworks.

the class HeartbeatServiceTest method testOneNodeTakesAllFlows.

@Test
public void testOneNodeTakesAllFlows() {
    List<Node> clusterNodes = getClusterNodes();
    // myself
    clusterNodes.get(0).setLastUpdated(200_000L);
    // set all nodes to failed except myself
    for (int i = 1; i < clusterNodes.size(); i++) {
        Node node = clusterNodes.get(i);
        node.setLastUpdated(50_000L);
    }
    when(nodeService.findAll()).thenReturn(clusterNodes);
    when(clock.getCurrentTimeMillis()).thenReturn(200_000L);
    // all flows that need to be re-distributed
    List<FlowLog> node1FlowLogs = getFlowLogs(2, 5000);
    List<String> suspendedFlows = node1FlowLogs.stream().map(FlowLog::getFlowId).distinct().collect(Collectors.toList());
    when(flowLogService.findAllByCloudbreakNodeId(NODE_1_ID)).thenReturn(new HashSet<>(node1FlowLogs));
    Set<FlowLog> node2FlowLogs = new HashSet<>(getFlowLogs(3, 3000));
    suspendedFlows.addAll(node2FlowLogs.stream().map(FlowLog::getFlowId).distinct().collect(Collectors.toList()));
    when(flowLogService.findAllByCloudbreakNodeId(NODE_2_ID)).thenReturn(node2FlowLogs);
    Map<Node, List<String>> distribution = new HashMap<>();
    distribution.computeIfAbsent(clusterNodes.get(0), v -> new ArrayList<>()).addAll(Arrays.asList(suspendedFlows.get(0), suspendedFlows.get(1), suspendedFlows.get(2), suspendedFlows.get(3), suspendedFlows.get(4)));
    when(flowDistributor.distribute(any(), any())).thenReturn(distribution);
    Set<FlowLog> myNewFlowLogs = new HashSet<>();
    myNewFlowLogs.addAll(node1FlowLogs);
    myNewFlowLogs.addAll(node2FlowLogs);
    when(flowLogService.findAllByCloudbreakNodeId(MY_ID)).thenReturn(myNewFlowLogs);
    when(runningFlows.get(any())).thenReturn(null);
    heartbeatService.scheduledFlowDistribution();
    verify(flowLogService).saveAll(flowLogListCaptor.capture());
    List<FlowLog> updatedFlows = flowLogListCaptor.getValue();
    assertEquals(myNewFlowLogs.size(), updatedFlows.size());
    for (FlowLog updatedFlow : updatedFlows) {
        assertEquals(MY_ID, updatedFlow.getCloudbreakNodeId());
    }
    verify(flow2Handler, times(5)).restartFlow(stringCaptor.capture());
    List<String> allFlowIds = stringCaptor.getAllValues();
    assertEquals(5L, allFlowIds.size());
    for (String flowId : suspendedFlows) {
        assertTrue(allFlowIds.contains(flowId));
    }
}