use of org.apache.tez.dag.app.rm.YarnTaskSchedulerService.HeldContainer in project tez by apache.
the class TestTaskScheduler method testTaskSchedulerPreemption.
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test(timeout = 5000)
public void testTaskSchedulerPreemption() throws Exception {
TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class);
Configuration conf = new Configuration();
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS, 3);
TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(DEFAULT_APP_HOST, DEFAULT_APP_PORT, DEFAULT_APP_URL, false, null, null, new PreemptionMatcher(), conf);
final TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp);
final TaskSchedulerWithDrainableContext scheduler = new TaskSchedulerWithDrainableContext(drainableAppCallback, mockRMClient);
scheduler.initialize();
RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class);
when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString())).thenReturn(mockRegResponse);
scheduler.start();
Resource totalResource = Resource.newInstance(4000, 4);
when(mockRMClient.getAvailableResources()).thenReturn(totalResource);
// no preemption
scheduler.getProgress();
drainableAppCallback.drain();
Assert.assertEquals(totalResource, scheduler.getTotalResources());
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
// allocate task
Object mockTask1 = mock(Object.class);
Object mockTask2 = mock(Object.class);
Object mockTask3 = mock(Object.class);
Object mockTask3Wait = mock(Object.class);
Object mockTask3Retry = mock(Object.class);
Object mockTask3KillA = mock(Object.class);
Object mockTask3KillB = mock(Object.class);
Object mockTaskPri8 = mock(Object.class);
Object obj3 = new Object();
Priority pri2 = Priority.newInstance(2);
Priority pri4 = Priority.newInstance(4);
Priority pri5 = Priority.newInstance(5);
Priority pri6 = Priority.newInstance(6);
Priority pri8 = Priority.newInstance(8);
ArgumentCaptor<CookieContainerRequest> requestCaptor = ArgumentCaptor.forClass(CookieContainerRequest.class);
final ArrayList<CookieContainerRequest> anyContainers = new ArrayList<CookieContainerRequest>();
Resource taskAsk = Resource.newInstance(1024, 1);
scheduler.allocateTask(mockTask1, taskAsk, null, null, pri2, null, null);
drainableAppCallback.drain();
verify(mockRMClient, times(1)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
scheduler.allocateTask(mockTask3, taskAsk, null, null, pri6, obj3, null);
drainableAppCallback.drain();
verify(mockRMClient, times(2)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
// later one in the allocation gets killed between the two task3's
scheduler.allocateTask(mockTask3KillA, taskAsk, null, null, pri6, obj3, null);
drainableAppCallback.drain();
verify(mockRMClient, times(3)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
// later one in the allocation gets killed between the two task3's
scheduler.allocateTask(mockTask3KillB, taskAsk, null, null, pri6, obj3, null);
drainableAppCallback.drain();
verify(mockRMClient, times(4)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
Resource freeResource = Resource.newInstance(500, 0);
when(mockRMClient.getAvailableResources()).thenReturn(freeResource);
scheduler.getProgress();
drainableAppCallback.drain();
Assert.assertEquals(totalResource, scheduler.getTotalResources());
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
final List<ArrayList<CookieContainerRequest>> anyList = new LinkedList<ArrayList<CookieContainerRequest>>();
final List<ArrayList<CookieContainerRequest>> emptyList = new LinkedList<ArrayList<CookieContainerRequest>>();
anyList.add(anyContainers);
List<Container> containers = new ArrayList<Container>();
Container mockContainer1 = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer1.getNodeId().getHost()).thenReturn("host1");
when(mockContainer1.getResource()).thenReturn(taskAsk);
when(mockContainer1.getPriority()).thenReturn(pri2);
ContainerId mockCId1 = mock(ContainerId.class);
when(mockContainer1.getId()).thenReturn(mockCId1);
containers.add(mockContainer1);
Container mockContainer2 = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer2.getNodeId().getHost()).thenReturn("host1");
when(mockContainer2.getResource()).thenReturn(taskAsk);
when(mockContainer2.getPriority()).thenReturn(pri6);
ContainerId mockCId2 = mock(ContainerId.class);
when(mockContainer2.getId()).thenReturn(mockCId2);
containers.add(mockContainer2);
Container mockContainer3A = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer3A.getNodeId().getHost()).thenReturn("host1");
when(mockContainer3A.getResource()).thenReturn(taskAsk);
when(mockContainer3A.getPriority()).thenReturn(pri6);
ContainerId mockCId3A = mock(ContainerId.class);
when(mockContainer3A.getId()).thenReturn(mockCId3A);
containers.add(mockContainer3A);
Container mockContainer3B = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer3B.getNodeId().getHost()).thenReturn("host1");
when(mockContainer3B.getResource()).thenReturn(taskAsk);
// high priority container
when(mockContainer3B.getPriority()).thenReturn(pri2);
ContainerId mockCId3B = mock(ContainerId.class);
when(mockContainer3B.getId()).thenReturn(mockCId3B);
containers.add(mockContainer3B);
when(mockRMClient.getMatchingRequests((Priority) any(), eq("host1"), (Resource) any())).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() {
@Override
public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable {
return emptyList;
}
});
// RackResolver by default puts hosts in default-rack
when(mockRMClient.getMatchingRequests((Priority) any(), eq("/default-rack"), (Resource) any())).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() {
@Override
public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable {
return emptyList;
}
});
when(mockRMClient.getMatchingRequests((Priority) any(), eq(ResourceRequest.ANY), (Resource) any())).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() {
int calls = 0;
@Override
public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable {
if (calls > 0) {
anyContainers.remove(0);
}
calls++;
return anyList;
}
});
Mockito.doAnswer(new Answer() {
public Object answer(InvocationOnMock invocation) {
Object[] args = invocation.getArguments();
ContainerId cId = (ContainerId) args[0];
scheduler.deallocateContainer(cId);
return null;
}
}).when(mockApp).preemptContainer((ContainerId) any());
scheduler.onContainersAllocated(containers);
drainableAppCallback.drain();
Assert.assertEquals(4, scheduler.taskAllocations.size());
Assert.assertEquals(4096, scheduler.allocatedResources.getMemory());
Assert.assertEquals(mockCId1, scheduler.taskAllocations.get(mockTask1).getId());
Assert.assertEquals(mockCId2, scheduler.taskAllocations.get(mockTask3).getId());
Assert.assertEquals(mockCId3A, scheduler.taskAllocations.get(mockTask3KillA).getId());
// high priority container assigned to lower pri task. This task should still be preempted
// because the task priority is relevant for preemption and not the container priority
Assert.assertEquals(mockCId3B, scheduler.taskAllocations.get(mockTask3KillB).getId());
// no preemption
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
// no need for task preemption until now - so they should match
Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption);
// add a pending request that cannot be allocated until resources free up
Object mockTask3WaitCookie = new Object();
scheduler.allocateTask(mockTask3Wait, taskAsk, null, null, pri6, obj3, mockTask3WaitCookie);
// add a pri 8 request for the pri 8 container that will not be matched
Object mockTaskPri8Cookie = new Object();
scheduler.allocateTask(mockTaskPri8, taskAsk, null, null, pri8, obj3, mockTaskPri8Cookie);
// no preemption - same pri
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(6)).addContainerRequest(requestCaptor.capture());
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
Container mockContainer4 = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer4.getNodeId().getHost()).thenReturn("host1");
when(mockContainer4.getResource()).thenReturn(taskAsk);
when(mockContainer4.getPriority()).thenReturn(pri8);
ContainerId mockCId4 = mock(ContainerId.class);
when(mockContainer4.getId()).thenReturn(mockCId4);
containers.clear();
containers.add(mockContainer4);
// new lower pri container added that wont be matched and eventually preempted
// Fudge new container being present in delayed allocation list due to race
HeldContainer heldContainer = new HeldContainer(mockContainer4, -1, -1, null, containerSignatureMatcher);
scheduler.delayedContainerManager.delayedContainers.add(heldContainer);
// no preemption - container assignment attempts < 3
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
// no need for task preemption until now - so they should match
Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption);
heldContainer.incrementAssignmentAttempts();
// no preemption - container assignment attempts < 3
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
heldContainer.incrementAssignmentAttempts();
heldContainer.incrementAssignmentAttempts();
// preemption - container released and resource asked again
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any());
verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId4);
// internally re-request pri8 task request because we release pri8 new container
verify(mockRMClient, times(7)).addContainerRequest(requestCaptor.capture());
CookieContainerRequest reAdded = requestCaptor.getValue();
Assert.assertEquals(pri8, reAdded.getPriority());
Assert.assertEquals(taskAsk, reAdded.getCapability());
Assert.assertEquals(mockTaskPri8Cookie, reAdded.getCookie().getAppCookie());
// remove fudging.
scheduler.delayedContainerManager.delayedContainers.clear();
// no need for task preemption until now - so they should match
Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption);
scheduler.allocateTask(mockTask3Retry, taskAsk, null, null, pri5, obj3, null);
// no preemption - higher pri. exact match
scheduler.getProgress();
// no need for task preemption until now - so they should match
drainableAppCallback.drain();
// no need for task preemption until now - so they should match
Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption);
verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any());
for (int i = 0; i < 11; ++i) {
scheduler.allocateTask(mockTask2, taskAsk, null, null, pri4, null, null);
}
drainableAppCallback.drain();
// mockTaskPri3KillB gets preempted to clear 10% of outstanding running preemptable tasks
// this is also a higher priority container than the pending task priority but was running a
// lower priority task. Task priority is relevant for preemption and not container priority as
// containers can run tasks of different priorities
// first heartbeat
scheduler.getProgress();
Assert.assertTrue(scheduler.numHeartbeats > scheduler.heartbeatAtLastPreemption);
drainableAppCallback.drain();
// second heartbeat
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any());
// third heartbeat
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any());
verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3B);
Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption);
// there are pending preemptions.
// first heartbeat
scheduler.getProgress();
// second heartbeat
scheduler.getProgress();
verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any());
// third heartbeat
scheduler.getProgress();
drainableAppCallback.drain();
// Next oldest mockTaskPri3KillA gets preempted to clear 10% of outstanding running preemptable tasks
verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any());
verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3A);
AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, "", DEFAULT_APP_URL);
when(mockApp.getFinalAppStatus()).thenReturn(finalStatus);
scheduler.shutdown();
drainableAppCallback.drain();
}
use of org.apache.tez.dag.app.rm.YarnTaskSchedulerService.HeldContainer in project tez by apache.
the class TestTaskScheduler method testTaskSchedulerDetermineMinHeldContainers.
@Test(timeout = 5000)
public void testTaskSchedulerDetermineMinHeldContainers() throws Exception {
TezAMRMClientAsync<CookieContainerRequest> mockRMClient = spy(new AMRMClientAsyncForTest(new AMRMClientForTest(), 100));
TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(DEFAULT_APP_HOST, DEFAULT_APP_PORT, DEFAULT_APP_URL, true, new Configuration());
final TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp);
TaskSchedulerWithDrainableContext scheduler = new TaskSchedulerWithDrainableContext(drainableAppCallback, mockRMClient);
scheduler.initialize();
scheduler.start();
String rack1 = "r1";
String rack2 = "r2";
String rack3 = "r3";
String node1Rack1 = "n1r1";
String node2Rack1 = "n2r1";
String node1Rack2 = "n1r2";
String node2Rack2 = "n2r2";
String node1Rack3 = "n1r3";
ApplicationAttemptId appId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0);
Resource r = Resource.newInstance(0, 0);
ContainerId mockCId1 = ContainerId.newInstance(appId, 0);
Container c1 = mock(Container.class, RETURNS_DEEP_STUBS);
// we are mocking directly
when(c1.getNodeId().getHost()).thenReturn("");
HeldContainer hc1 = Mockito.spy(new HeldContainer(c1, 0, 0, null, containerSignatureMatcher));
when(hc1.getNode()).thenReturn(node1Rack1);
when(hc1.getRack()).thenReturn(rack1);
when(c1.getId()).thenReturn(mockCId1);
when(c1.getResource()).thenReturn(r);
when(hc1.getContainer()).thenReturn(c1);
ContainerId mockCId2 = ContainerId.newInstance(appId, 1);
Container c2 = mock(Container.class, RETURNS_DEEP_STUBS);
// we are mocking directly
when(c2.getNodeId().getHost()).thenReturn("");
HeldContainer hc2 = Mockito.spy(new HeldContainer(c2, 0, 0, null, containerSignatureMatcher));
when(hc2.getNode()).thenReturn(node2Rack1);
when(hc2.getRack()).thenReturn(rack1);
when(c2.getId()).thenReturn(mockCId2);
when(c2.getResource()).thenReturn(r);
when(hc2.getContainer()).thenReturn(c2);
ContainerId mockCId3 = ContainerId.newInstance(appId, 2);
Container c3 = mock(Container.class, RETURNS_DEEP_STUBS);
// we are mocking directly
when(c3.getNodeId().getHost()).thenReturn("");
HeldContainer hc3 = Mockito.spy(new HeldContainer(c3, 0, 0, null, containerSignatureMatcher));
when(hc3.getNode()).thenReturn(node1Rack1);
when(hc3.getRack()).thenReturn(rack1);
when(c3.getId()).thenReturn(mockCId3);
when(c3.getResource()).thenReturn(r);
when(hc3.getContainer()).thenReturn(c3);
ContainerId mockCId4 = ContainerId.newInstance(appId, 3);
Container c4 = mock(Container.class, RETURNS_DEEP_STUBS);
// we are mocking directly
when(c4.getNodeId().getHost()).thenReturn("");
HeldContainer hc4 = Mockito.spy(new HeldContainer(c4, 0, 0, null, containerSignatureMatcher));
when(hc4.getNode()).thenReturn(node2Rack1);
when(hc4.getRack()).thenReturn(rack1);
when(c4.getId()).thenReturn(mockCId4);
when(c4.getResource()).thenReturn(r);
when(hc4.getContainer()).thenReturn(c4);
ContainerId mockCId5 = ContainerId.newInstance(appId, 4);
Container c5 = mock(Container.class, RETURNS_DEEP_STUBS);
// we are mocking directly
when(c5.getNodeId().getHost()).thenReturn("");
HeldContainer hc5 = Mockito.spy(new HeldContainer(c5, 0, 0, null, containerSignatureMatcher));
when(hc5.getNode()).thenReturn(node1Rack2);
when(hc5.getRack()).thenReturn(rack2);
when(c5.getId()).thenReturn(mockCId5);
when(c5.getResource()).thenReturn(r);
when(hc5.getContainer()).thenReturn(c5);
ContainerId mockCId6 = ContainerId.newInstance(appId, 5);
Container c6 = mock(Container.class, RETURNS_DEEP_STUBS);
// we are mocking directly
when(c6.getNodeId().getHost()).thenReturn("");
HeldContainer hc6 = Mockito.spy(new HeldContainer(c6, 0, 0, null, containerSignatureMatcher));
when(hc6.getNode()).thenReturn(node2Rack2);
when(hc6.getRack()).thenReturn(rack2);
when(c6.getId()).thenReturn(mockCId6);
when(c6.getResource()).thenReturn(r);
when(hc6.getContainer()).thenReturn(c6);
ContainerId mockCId7 = ContainerId.newInstance(appId, 6);
Container c7 = mock(Container.class, RETURNS_DEEP_STUBS);
// we are mocking directly
when(c7.getNodeId().getHost()).thenReturn("");
HeldContainer hc7 = Mockito.spy(new HeldContainer(c7, 0, 0, null, containerSignatureMatcher));
when(hc7.getNode()).thenReturn(node1Rack3);
when(hc7.getRack()).thenReturn(rack3);
when(c7.getId()).thenReturn(mockCId7);
when(c7.getResource()).thenReturn(r);
when(hc7.getContainer()).thenReturn(c7);
scheduler.heldContainers.put(mockCId1, hc1);
scheduler.heldContainers.put(mockCId2, hc2);
scheduler.heldContainers.put(mockCId3, hc3);
scheduler.heldContainers.put(mockCId4, hc4);
scheduler.heldContainers.put(mockCId5, hc5);
scheduler.heldContainers.put(mockCId6, hc6);
scheduler.heldContainers.put(mockCId7, hc7);
// test empty case
scheduler.sessionNumMinHeldContainers = 0;
scheduler.determineMinHeldContainers();
Assert.assertEquals(0, scheduler.sessionMinHeldContainers.size());
// test min >= held
scheduler.sessionNumMinHeldContainers = 7;
scheduler.determineMinHeldContainers();
Assert.assertEquals(7, scheduler.sessionMinHeldContainers.size());
// test min < held
scheduler.sessionNumMinHeldContainers = 5;
scheduler.determineMinHeldContainers();
Assert.assertEquals(5, scheduler.sessionMinHeldContainers.size());
Set<HeldContainer> heldContainers = Sets.newHashSet();
for (ContainerId cId : scheduler.sessionMinHeldContainers) {
heldContainers.add(scheduler.heldContainers.get(cId));
}
Set<String> racks = Sets.newHashSet();
Set<String> nodes = Sets.newHashSet();
for (HeldContainer hc : heldContainers) {
nodes.add(hc.getNode());
racks.add(hc.getRack());
}
// 1 container from each node in rack1 and rack2. 1 container from rack3.
// covers not enough containers in rack (rack 3)
// covers just enough containers in rack (rack 2)
// covers more than enough containers in rack (rack 1)
Assert.assertEquals(5, nodes.size());
Assert.assertTrue(nodes.contains(node1Rack1) && nodes.contains(node2Rack1) && nodes.contains(node1Rack2) && nodes.contains(node2Rack2) && nodes.contains(node1Rack3));
Assert.assertEquals(3, racks.size());
Assert.assertTrue(racks.contains(rack1) && racks.contains(rack2) && racks.contains(rack3));
long currTime = System.currentTimeMillis();
heldContainers.clear();
heldContainers.addAll(scheduler.heldContainers.values());
for (HeldContainer hc : heldContainers) {
when(hc.isNew()).thenReturn(true);
scheduler.delayedContainerManager.addDelayedContainer(hc.getContainer(), currTime);
}
Thread.sleep(1000);
drainableAppCallback.drain();
// only the 2 container not in min-held containers are released
verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any());
Assert.assertEquals(5, scheduler.heldContainers.size());
AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, SUCCEED_APP_MESSAGE, DEFAULT_APP_URL);
when(mockApp.getFinalAppStatus()).thenReturn(finalStatus);
scheduler.shutdown();
}
use of org.apache.tez.dag.app.rm.YarnTaskSchedulerService.HeldContainer in project tez by apache.
the class TestTaskScheduler method testTaskSchedulerHeldContainersReleaseAfterExpired.
@Test(timeout = 3000)
public void testTaskSchedulerHeldContainersReleaseAfterExpired() throws Exception {
final TezAMRMClientAsync<CookieContainerRequest> mockRMClient = spy(new AMRMClientAsyncForTest(new AMRMClientForTest(), 100));
final TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(DEFAULT_APP_HOST, DEFAULT_APP_PORT, DEFAULT_APP_URL, true, new Configuration());
final TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp);
final TaskSchedulerWithDrainableContext scheduler = new TaskSchedulerWithDrainableContext(drainableAppCallback, mockRMClient);
scheduler.initialize();
scheduler.start();
ApplicationAttemptId appId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0);
ContainerId containerId = ContainerId.newInstance(appId, 0);
Container c1 = mock(Container.class, RETURNS_DEEP_STUBS);
// we are mocking directly
when(c1.getNodeId().getHost()).thenReturn("");
HeldContainer hc1 = mock(HeldContainer.class);
when(c1.getId()).thenReturn(containerId);
when(hc1.getContainer()).thenReturn(c1);
when(hc1.isNew()).thenReturn(false);
// containerExpiryTime = 0
scheduler.heldContainers.put(containerId, hc1);
long currTime = System.currentTimeMillis();
scheduler.delayedContainerManager.addDelayedContainer(hc1.getContainer(), currTime);
// sleep and wait for mainLoop() check-in to release this expired held container.
Thread.sleep(1000);
verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any());
Assert.assertEquals(0, scheduler.heldContainers.size());
AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, SUCCEED_APP_MESSAGE, DEFAULT_APP_URL);
when(mockApp.getFinalAppStatus()).thenReturn(finalStatus);
scheduler.shutdown();
}
Aggregations