Search in sources :

Example 1 with ApplicationFlowInformation

use of com.sequenceiq.flow.core.ApplicationFlowInformation in project cloudbreak by hortonworks.

the class HeartbeatServiceTest method testOneNodeTakesAllFlowsWithTerminationFlowShouldBeDistributed.

@Test
public void testOneNodeTakesAllFlowsWithTerminationFlowShouldBeDistributed() {
    List<Node> clusterNodes = getClusterNodes();
    // myself
    clusterNodes.get(0).setLastUpdated(200_000L);
    // set all nodes to failed except myself
    for (int i = 1; i < clusterNodes.size(); i++) {
        Node node = clusterNodes.get(i);
        node.setLastUpdated(50_000L);
    }
    when(nodeService.findAll()).thenReturn(clusterNodes);
    when(clock.getCurrentTimeMillis()).thenReturn(200_000L);
    // all flows that need to be re-distributed
    List<FlowLog> node1FlowLogs = getFlowLogs(2, 5000);
    node1FlowLogs.forEach(fl -> fl.setFlowType(ClassValue.of(HelloWorldFlowConfig.class)));
    List<String> suspendedFlows = node1FlowLogs.stream().map(FlowLog::getFlowId).distinct().collect(Collectors.toList());
    when(flowLogService.findAllByCloudbreakNodeId(NODE_1_ID)).thenReturn(new HashSet<>(node1FlowLogs));
    Set<FlowLog> node2FlowLogs = new HashSet<>(getFlowLogs(3, 3000));
    node2FlowLogs.forEach(fl -> fl.setFlowType(ClassValue.of(HelloWorldFlowConfig.class)));
    suspendedFlows.addAll(node2FlowLogs.stream().map(FlowLog::getFlowId).distinct().collect(Collectors.toList()));
    when(flowLogService.findAllByCloudbreakNodeId(NODE_2_ID)).thenReturn(node2FlowLogs);
    Map<Node, List<String>> distribution = new HashMap<>();
    distribution.computeIfAbsent(clusterNodes.get(0), v -> new ArrayList<>()).addAll(Arrays.asList(suspendedFlows.get(0), suspendedFlows.get(1), suspendedFlows.get(2), suspendedFlows.get(3), suspendedFlows.get(4)));
    when(flowDistributor.distribute(any(), any())).thenReturn(distribution);
    Set<FlowLog> myNewFlowLogs = new HashSet<>();
    myNewFlowLogs.addAll(node1FlowLogs);
    myNewFlowLogs.addAll(node2FlowLogs);
    when(flowLogService.findAllByCloudbreakNodeId(MY_ID)).thenReturn(myNewFlowLogs);
    when(runningFlows.get(any())).thenReturn(null);
    List<Long> stackIds = myNewFlowLogs.stream().map(FlowLog::getResourceId).distinct().collect(Collectors.toList());
    when(haApplication.getDeletingResources(anySet())).thenReturn(Set.of(stackIds.get(0), stackIds.get(2)));
    doReturn(Collections.singletonList(HelloWorldFlowConfig.class)).when(applicationFlowInformation).getTerminationFlow();
    heartbeatService.scheduledFlowDistribution();
    verify(flowLogService).saveAll(flowLogListCaptor.capture());
    List<FlowLog> updatedFlows = flowLogListCaptor.getValue();
    assertEquals(myNewFlowLogs.size(), updatedFlows.size());
    for (FlowLog updatedFlow : updatedFlows) {
        assertEquals(MY_ID, updatedFlow.getCloudbreakNodeId());
    }
    verify(flow2Handler, times(5)).restartFlow(stringCaptor.capture());
    List<String> allFlowIds = stringCaptor.getAllValues();
    assertEquals(5L, allFlowIds.size());
    for (String flowId : suspendedFlows) {
        assertTrue(allFlowIds.contains(flowId));
    }
}
Also used : Arrays(java.util.Arrays) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) Random(java.util.Random) HaApplication(com.sequenceiq.cloudbreak.service.ha.HaApplication) SecureRandom(java.security.SecureRandom) TransactionService(com.sequenceiq.cloudbreak.common.service.TransactionService) OptimisticLockingFailureException(org.springframework.dao.OptimisticLockingFailureException) InMemoryCleanup(com.sequenceiq.flow.cleanup.InMemoryCleanup) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Map(java.util.Map) ZoneOffset(java.time.ZoneOffset) Mockito.doReturn(org.mockito.Mockito.doReturn) FlowLog(com.sequenceiq.flow.domain.FlowLog) Node(com.sequenceiq.cloudbreak.ha.domain.Node) Collection(java.util.Collection) Set(java.util.Set) UUID(java.util.UUID) ArgumentMatchers.anyList(org.mockito.ArgumentMatchers.anyList) Collectors(java.util.stream.Collectors) HeartbeatService(com.sequenceiq.cloudbreak.service.ha.HeartbeatService) List(java.util.List) TransactionExecutionException(com.sequenceiq.cloudbreak.common.service.TransactionService.TransactionExecutionException) ClassValue(com.sequenceiq.flow.domain.ClassValue) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) FlowRegister(com.sequenceiq.flow.core.FlowRegister) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) IntStream(java.util.stream.IntStream) Clock(com.sequenceiq.cloudbreak.common.service.Clock) NodeService(com.sequenceiq.cloudbreak.ha.service.NodeService) ArgumentMatchers.anySet(org.mockito.ArgumentMatchers.anySet) Mock(org.mockito.Mock) LocalDateTime(java.time.LocalDateTime) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) Captor(org.mockito.Captor) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ArgumentCaptor(org.mockito.ArgumentCaptor) ApplicationFlowInformation(com.sequenceiq.flow.core.ApplicationFlowInformation) OperationType(com.sequenceiq.flow.api.model.operation.OperationType) FlowLogService(com.sequenceiq.flow.core.FlowLogService) StateStatus(com.sequenceiq.flow.domain.StateStatus) Before(org.junit.Before) InjectMocks(org.mockito.InjectMocks) Flow2Handler(com.sequenceiq.flow.core.Flow2Handler) HelloWorldFlowConfig(com.sequenceiq.flow.core.helloworld.config.HelloWorldFlowConfig) InMemoryStateStore(com.sequenceiq.cloudbreak.cloud.store.InMemoryStateStore) Assert.assertTrue(org.junit.Assert.assertTrue) ReflectionTestUtils(org.springframework.test.util.ReflectionTestUtils) Mockito.times(org.mockito.Mockito.times) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Mockito.verify(org.mockito.Mockito.verify) ArgumentMatchers.anyCollection(org.mockito.ArgumentMatchers.anyCollection) Assert.assertNull(org.junit.Assert.assertNull) FlowDistributor(com.sequenceiq.cloudbreak.ha.service.FlowDistributor) NodeConfig(com.sequenceiq.flow.ha.NodeConfig) FlowConfiguration(com.sequenceiq.flow.core.config.FlowConfiguration) PollGroup(com.sequenceiq.cloudbreak.cloud.scheduler.PollGroup) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) FlowLog(com.sequenceiq.flow.domain.FlowLog) HashMap(java.util.HashMap) Node(com.sequenceiq.cloudbreak.ha.domain.Node) ArrayList(java.util.ArrayList) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) ArgumentMatchers.anyList(org.mockito.ArgumentMatchers.anyList) List(java.util.List) ArrayList(java.util.ArrayList) HelloWorldFlowConfig(com.sequenceiq.flow.core.helloworld.config.HelloWorldFlowConfig) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 2 with ApplicationFlowInformation

use of com.sequenceiq.flow.core.ApplicationFlowInformation in project cloudbreak by hortonworks.

the class HeartbeatServiceTest method testOneNodeTakesAllFlowsWithInvalidFlows.

@Test
public void testOneNodeTakesAllFlowsWithInvalidFlows() {
    List<Node> clusterNodes = getClusterNodes();
    // myself
    clusterNodes.get(0).setLastUpdated(200_000L);
    // set all nodes to failed except myself
    for (int i = 1; i < clusterNodes.size(); i++) {
        Node node = clusterNodes.get(i);
        node.setLastUpdated(50_000L);
    }
    when(nodeService.findAll()).thenReturn(clusterNodes);
    when(clock.getCurrentTimeMillis()).thenReturn(200_000L);
    // all flows that need to be re-distributed
    List<FlowLog> node1FlowLogs = getFlowLogs(2, 5000);
    List<String> suspendedFlows = node1FlowLogs.stream().map(FlowLog::getFlowId).distinct().collect(Collectors.toList());
    when(flowLogService.findAllByCloudbreakNodeId(NODE_1_ID)).thenReturn(new HashSet<>(node1FlowLogs));
    Set<FlowLog> node2FlowLogs = new HashSet<>(getFlowLogs(3, 3000));
    suspendedFlows.addAll(node2FlowLogs.stream().map(FlowLog::getFlowId).distinct().collect(Collectors.toList()));
    when(flowLogService.findAllByCloudbreakNodeId(NODE_2_ID)).thenReturn(node2FlowLogs);
    Map<Node, List<String>> distribution = new HashMap<>();
    distribution.computeIfAbsent(clusterNodes.get(0), v -> new ArrayList<>()).addAll(Arrays.asList(suspendedFlows.get(0), suspendedFlows.get(1), suspendedFlows.get(2), suspendedFlows.get(3), suspendedFlows.get(4)));
    when(flowDistributor.distribute(any(), any())).thenReturn(distribution);
    Set<FlowLog> myNewFlowLogs = new HashSet<>();
    myNewFlowLogs.addAll(node1FlowLogs);
    myNewFlowLogs.addAll(node2FlowLogs);
    when(flowLogService.findAllByCloudbreakNodeId(MY_ID)).thenReturn(myNewFlowLogs);
    when(runningFlows.get(any())).thenReturn(null);
    List<Long> stackIds = myNewFlowLogs.stream().map(FlowLog::getResourceId).distinct().collect(Collectors.toList());
    when(haApplication.getDeletingResources(anySet())).thenReturn(Set.of(stackIds.get(0), stackIds.get(2)));
    doReturn(Collections.singletonList(HelloWorldFlowConfig.class)).when(applicationFlowInformation).getTerminationFlow();
    List<FlowLog> invalidFlowLogs = myNewFlowLogs.stream().filter(fl -> fl.getResourceId().equals(stackIds.get(0)) || fl.getResourceId().equals(stackIds.get(2))).collect(Collectors.toList());
    heartbeatService.scheduledFlowDistribution();
    verify(flowLogService).saveAll(flowLogListCaptor.capture());
    List<FlowLog> updatedFlows = flowLogListCaptor.getValue();
    assertEquals(myNewFlowLogs.size(), updatedFlows.size());
    for (FlowLog updatedFlow : updatedFlows) {
        if (invalidFlowLogs.contains(updatedFlow)) {
            assertEquals(StateStatus.SUCCESSFUL, updatedFlow.getStateStatus());
            assertNull(updatedFlow.getCloudbreakNodeId());
        } else {
            assertEquals(MY_ID, updatedFlow.getCloudbreakNodeId());
        }
    }
    verify(flow2Handler, times(5)).restartFlow(stringCaptor.capture());
    List<String> allFlowIds = stringCaptor.getAllValues();
    assertEquals(5L, allFlowIds.size());
    for (String flowId : suspendedFlows) {
        assertTrue(allFlowIds.contains(flowId));
    }
}
Also used : Arrays(java.util.Arrays) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) Random(java.util.Random) HaApplication(com.sequenceiq.cloudbreak.service.ha.HaApplication) SecureRandom(java.security.SecureRandom) TransactionService(com.sequenceiq.cloudbreak.common.service.TransactionService) OptimisticLockingFailureException(org.springframework.dao.OptimisticLockingFailureException) InMemoryCleanup(com.sequenceiq.flow.cleanup.InMemoryCleanup) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Map(java.util.Map) ZoneOffset(java.time.ZoneOffset) Mockito.doReturn(org.mockito.Mockito.doReturn) FlowLog(com.sequenceiq.flow.domain.FlowLog) Node(com.sequenceiq.cloudbreak.ha.domain.Node) Collection(java.util.Collection) Set(java.util.Set) UUID(java.util.UUID) ArgumentMatchers.anyList(org.mockito.ArgumentMatchers.anyList) Collectors(java.util.stream.Collectors) HeartbeatService(com.sequenceiq.cloudbreak.service.ha.HeartbeatService) List(java.util.List) TransactionExecutionException(com.sequenceiq.cloudbreak.common.service.TransactionService.TransactionExecutionException) ClassValue(com.sequenceiq.flow.domain.ClassValue) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) FlowRegister(com.sequenceiq.flow.core.FlowRegister) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) IntStream(java.util.stream.IntStream) Clock(com.sequenceiq.cloudbreak.common.service.Clock) NodeService(com.sequenceiq.cloudbreak.ha.service.NodeService) ArgumentMatchers.anySet(org.mockito.ArgumentMatchers.anySet) Mock(org.mockito.Mock) LocalDateTime(java.time.LocalDateTime) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) Captor(org.mockito.Captor) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ArgumentCaptor(org.mockito.ArgumentCaptor) ApplicationFlowInformation(com.sequenceiq.flow.core.ApplicationFlowInformation) OperationType(com.sequenceiq.flow.api.model.operation.OperationType) FlowLogService(com.sequenceiq.flow.core.FlowLogService) StateStatus(com.sequenceiq.flow.domain.StateStatus) Before(org.junit.Before) InjectMocks(org.mockito.InjectMocks) Flow2Handler(com.sequenceiq.flow.core.Flow2Handler) HelloWorldFlowConfig(com.sequenceiq.flow.core.helloworld.config.HelloWorldFlowConfig) InMemoryStateStore(com.sequenceiq.cloudbreak.cloud.store.InMemoryStateStore) Assert.assertTrue(org.junit.Assert.assertTrue) ReflectionTestUtils(org.springframework.test.util.ReflectionTestUtils) Mockito.times(org.mockito.Mockito.times) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Mockito.verify(org.mockito.Mockito.verify) ArgumentMatchers.anyCollection(org.mockito.ArgumentMatchers.anyCollection) Assert.assertNull(org.junit.Assert.assertNull) FlowDistributor(com.sequenceiq.cloudbreak.ha.service.FlowDistributor) NodeConfig(com.sequenceiq.flow.ha.NodeConfig) FlowConfiguration(com.sequenceiq.flow.core.config.FlowConfiguration) PollGroup(com.sequenceiq.cloudbreak.cloud.scheduler.PollGroup) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) FlowLog(com.sequenceiq.flow.domain.FlowLog) HashMap(java.util.HashMap) Node(com.sequenceiq.cloudbreak.ha.domain.Node) ArrayList(java.util.ArrayList) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) ArgumentMatchers.anyList(org.mockito.ArgumentMatchers.anyList) List(java.util.List) ArrayList(java.util.ArrayList) HelloWorldFlowConfig(com.sequenceiq.flow.core.helloworld.config.HelloWorldFlowConfig) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 3 with ApplicationFlowInformation

use of com.sequenceiq.flow.core.ApplicationFlowInformation in project cloudbreak by hortonworks.

the class EventBusConfigTest method uncaughtErrorButFlowNotFound.

@Test
@DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD)
public void uncaughtErrorButFlowNotFound() {
    FlowLog flowLog = new FlowLog();
    flowLog.setFlowId("123");
    when(flowLogDBService.getLastFlowLog("123")).thenReturn(Optional.empty());
    Event.Headers headers = new Event.Headers();
    headers.set("FLOW_ID", "123");
    eventBus.on(Selectors.regex("exampleselector"), (Consumer<Event<? extends Payload>>) event -> {
        throw new RuntimeException("uncaught exception");
    });
    eventBus.notify("exampleselector", new Event<>(headers, null));
    verify(flowLogDBService, timeout(2000).times(1)).getLastFlowLog("123");
    verify(applicationFlowInformation, times(0)).handleFlowFail(any());
    verify(flowLogDBService, times(0)).updateLastFlowLogStatus(any(), anyBoolean());
    verify(flowLogDBService, times(0)).finalize(any());
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) DirtiesContext(org.springframework.test.annotation.DirtiesContext) FlowLog(com.sequenceiq.flow.domain.FlowLog) Payload(com.sequenceiq.cloudbreak.common.event.Payload) RunWith(org.junit.runner.RunWith) Mockito.times(org.mockito.Mockito.times) EventBus(reactor.bus.EventBus) Mockito.when(org.mockito.Mockito.when) ArgumentMatchers.anyBoolean(org.mockito.ArgumentMatchers.anyBoolean) Mockito.verify(org.mockito.Mockito.verify) Inject(javax.inject.Inject) Test(org.junit.jupiter.api.Test) Mockito.timeout(org.mockito.Mockito.timeout) Selectors(reactor.bus.selector.Selectors) ApplicationFlowInformation(com.sequenceiq.flow.core.ApplicationFlowInformation) SpringBootTest(org.springframework.boot.test.context.SpringBootTest) Event(reactor.bus.Event) Optional(java.util.Optional) FlowLogDBService(com.sequenceiq.flow.service.flowlog.FlowLogDBService) SpringRunner(org.springframework.test.context.junit4.SpringRunner) Consumer(reactor.fn.Consumer) MockBean(org.springframework.boot.test.mock.mockito.MockBean) FlowLog(com.sequenceiq.flow.domain.FlowLog) Event(reactor.bus.Event) Payload(com.sequenceiq.cloudbreak.common.event.Payload) Test(org.junit.jupiter.api.Test) SpringBootTest(org.springframework.boot.test.context.SpringBootTest) DirtiesContext(org.springframework.test.annotation.DirtiesContext)

Example 4 with ApplicationFlowInformation

use of com.sequenceiq.flow.core.ApplicationFlowInformation in project cloudbreak by hortonworks.

the class EventBusConfigTest method uncaughtErrorButFlowFound.

@Test
@DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD)
public void uncaughtErrorButFlowFound() {
    FlowLog flowLog = new FlowLog();
    flowLog.setFlowId("123");
    when(flowLogDBService.getLastFlowLog("123")).thenReturn(Optional.of(flowLog));
    Event.Headers headers = new Event.Headers();
    headers.set("FLOW_ID", "123");
    eventBus.on(Selectors.regex("exampleselector"), (Consumer<Event<? extends Payload>>) event -> {
        throw new RuntimeException("uncaught exception");
    });
    eventBus.notify("exampleselector", new Event<>(headers, null));
    verify(flowLogDBService, timeout(2000).times(1)).getLastFlowLog("123");
    verify(applicationFlowInformation, timeout(1000).times(1)).handleFlowFail(flowLog);
    verify(flowLogDBService, timeout(1000).times(1)).updateLastFlowLogStatus(flowLog, true);
    verify(flowLogDBService, timeout(1000).times(1)).finalize(flowLog.getFlowId());
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) DirtiesContext(org.springframework.test.annotation.DirtiesContext) FlowLog(com.sequenceiq.flow.domain.FlowLog) Payload(com.sequenceiq.cloudbreak.common.event.Payload) RunWith(org.junit.runner.RunWith) Mockito.times(org.mockito.Mockito.times) EventBus(reactor.bus.EventBus) Mockito.when(org.mockito.Mockito.when) ArgumentMatchers.anyBoolean(org.mockito.ArgumentMatchers.anyBoolean) Mockito.verify(org.mockito.Mockito.verify) Inject(javax.inject.Inject) Test(org.junit.jupiter.api.Test) Mockito.timeout(org.mockito.Mockito.timeout) Selectors(reactor.bus.selector.Selectors) ApplicationFlowInformation(com.sequenceiq.flow.core.ApplicationFlowInformation) SpringBootTest(org.springframework.boot.test.context.SpringBootTest) Event(reactor.bus.Event) Optional(java.util.Optional) FlowLogDBService(com.sequenceiq.flow.service.flowlog.FlowLogDBService) SpringRunner(org.springframework.test.context.junit4.SpringRunner) Consumer(reactor.fn.Consumer) MockBean(org.springframework.boot.test.mock.mockito.MockBean) FlowLog(com.sequenceiq.flow.domain.FlowLog) Event(reactor.bus.Event) Payload(com.sequenceiq.cloudbreak.common.event.Payload) Test(org.junit.jupiter.api.Test) SpringBootTest(org.springframework.boot.test.context.SpringBootTest) DirtiesContext(org.springframework.test.annotation.DirtiesContext)

Aggregations

ApplicationFlowInformation (com.sequenceiq.flow.core.ApplicationFlowInformation)4 FlowLog (com.sequenceiq.flow.domain.FlowLog)4 RunWith (org.junit.runner.RunWith)4 ArgumentMatchers.any (org.mockito.ArgumentMatchers.any)4 Mockito.times (org.mockito.Mockito.times)4 Mockito.verify (org.mockito.Mockito.verify)4 Mockito.when (org.mockito.Mockito.when)4 PollGroup (com.sequenceiq.cloudbreak.cloud.scheduler.PollGroup)2 InMemoryStateStore (com.sequenceiq.cloudbreak.cloud.store.InMemoryStateStore)2 Payload (com.sequenceiq.cloudbreak.common.event.Payload)2 Clock (com.sequenceiq.cloudbreak.common.service.Clock)2 TransactionService (com.sequenceiq.cloudbreak.common.service.TransactionService)2 TransactionExecutionException (com.sequenceiq.cloudbreak.common.service.TransactionService.TransactionExecutionException)2 Node (com.sequenceiq.cloudbreak.ha.domain.Node)2 FlowDistributor (com.sequenceiq.cloudbreak.ha.service.FlowDistributor)2 NodeService (com.sequenceiq.cloudbreak.ha.service.NodeService)2 HaApplication (com.sequenceiq.cloudbreak.service.ha.HaApplication)2 HeartbeatService (com.sequenceiq.cloudbreak.service.ha.HeartbeatService)2 OperationType (com.sequenceiq.flow.api.model.operation.OperationType)2 InMemoryCleanup (com.sequenceiq.flow.cleanup.InMemoryCleanup)2