use of org.apache.tez.serviceplugins.api.ContainerLaunchRequest in project tez by apache.
the class TestContainerLauncherManager method testEventRouting.
@Test(timeout = 5000)
public void testEventRouting() throws Exception {
Configuration conf = new Configuration(false);
UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
AppContext appContext = mock(AppContext.class);
TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
String customLauncherName = "customLauncher";
List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
ByteBuffer bb = ByteBuffer.allocate(4);
bb.putInt(0, 3);
UserPayload customPayload = UserPayload.create(bb);
launcherDescriptors.add(new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName()).setUserPayload(customPayload));
launcherDescriptors.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload));
ContainerLaucherRouterForMultipleLauncherTest clr = new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, launcherDescriptors, true);
try {
clr.init(conf);
clr.start();
assertEquals(2, clr.getNumContainerLaunchers());
assertTrue(clr.getYarnContainerLauncherCreated());
assertFalse(clr.getUberContainerLauncherCreated());
assertEquals(customLauncherName, clr.getContainerLauncherName(0));
assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
verify(clr.getTestContainerLauncher(0)).initialize();
verify(clr.getTestContainerLauncher(0)).start();
verify(clr.getTestContainerLauncher(1)).initialize();
verify(clr.getTestContainerLauncher(1)).start();
ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class);
Container container1 = mock(Container.class);
ContainerLaunchContext clc2 = mock(ContainerLaunchContext.class);
Container container2 = mock(Container.class);
ContainerLauncherLaunchRequestEvent launchRequestEvent1 = new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0);
ContainerLauncherLaunchRequestEvent launchRequestEvent2 = new ContainerLauncherLaunchRequestEvent(clc2, container2, 1, 0, 0);
clr.handle(launchRequestEvent1);
ArgumentCaptor<ContainerLaunchRequest> captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class);
verify(clr.getTestContainerLauncher(0)).launchContainer(captor.capture());
assertEquals(1, captor.getAllValues().size());
ContainerLaunchRequest launchRequest1 = captor.getValue();
assertEquals(clc1, launchRequest1.getContainerLaunchContext());
clr.handle(launchRequestEvent2);
captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class);
verify(clr.getTestContainerLauncher(1)).launchContainer(captor.capture());
assertEquals(1, captor.getAllValues().size());
ContainerLaunchRequest launchRequest2 = captor.getValue();
assertEquals(clc2, launchRequest2.getContainerLaunchContext());
} finally {
clr.stop();
verify(clr.getTestContainerLauncher(0)).shutdown();
verify(clr.getTestContainerLauncher(1)).shutdown();
}
}
use of org.apache.tez.serviceplugins.api.ContainerLaunchRequest in project tez by apache.
the class TestMockDAGAppMaster method testConcurrencyLimit.
@Test(timeout = 100000)
public void testConcurrencyLimit() throws Exception {
// the test relies on local mode behavior of launching a new container per task.
// so task concurrency == container concurrency
TezConfiguration tezconf = new TezConfiguration(defaultConf);
final int concurrencyLimit = 5;
MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false, concurrencyLimit * 4, 1000);
tezClient.start();
MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
mockLauncher.startScheduling(false);
final AtomicInteger concurrency = new AtomicInteger(0);
final AtomicBoolean exceededConcurrency = new AtomicBoolean(false);
mockApp.containerDelegate = new ContainerDelegate() {
@Override
public void stop(ContainerStopRequest event) {
concurrency.decrementAndGet();
}
@Override
public void launch(ContainerLaunchRequest event) {
int maxConc = concurrency.incrementAndGet();
if (maxConc > concurrencyLimit) {
exceededConcurrency.set(true);
}
System.out.println("Launched: " + maxConc);
}
};
DAG dag = DAG.create("testConcurrencyLimit");
Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 20).setConf(TezConfiguration.TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY, String.valueOf(concurrencyLimit));
dag.addVertex(vA);
mockLauncher.startScheduling(true);
DAGClient dagClient = tezClient.submitDAG(dag);
dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
Assert.assertFalse(exceededConcurrency.get());
tezClient.stop();
}
use of org.apache.tez.serviceplugins.api.ContainerLaunchRequest in project tez by apache.
the class ContainerLauncherManager method handle.
@Override
public void handle(ContainerLauncherEvent event) {
int launcherId = event.getLauncherId();
String schedulerName = appContext.getTaskSchedulerName(event.getSchedulerId());
String taskCommName = appContext.getTaskCommunicatorName(event.getTaskCommId());
switch(event.getType()) {
case CONTAINER_LAUNCH_REQUEST:
ContainerLauncherLaunchRequestEvent launchEvent = (ContainerLauncherLaunchRequestEvent) event;
ContainerLaunchRequest launchRequest = new ContainerLaunchRequest(launchEvent.getNodeId(), launchEvent.getContainerId(), launchEvent.getContainerToken(), launchEvent.getContainerLaunchContext(), launchEvent.getContainer(), schedulerName, taskCommName);
try {
containerLaunchers[launcherId].launchContainer(launchRequest);
} catch (Exception e) {
String msg = "Error when launching container" + ", containerLauncher=" + Utils.getContainerLauncherIdentifierString(launcherId, appContext) + ", scheduler=" + Utils.getTaskSchedulerIdentifierString(event.getSchedulerId(), appContext) + ", taskCommunicator=" + Utils.getTaskCommIdentifierString(event.getTaskCommId(), appContext);
LOG.error(msg, e);
sendEvent(new DAGAppMasterEventUserServiceFatalError(DAGAppMasterEventType.CONTAINER_LAUNCHER_SERVICE_FATAL_ERROR, msg, e));
}
break;
case CONTAINER_STOP_REQUEST:
ContainerStopRequest stopRequest = new ContainerStopRequest(event.getNodeId(), event.getContainerId(), event.getContainerToken(), schedulerName, taskCommName);
try {
containerLaunchers[launcherId].stopContainer(stopRequest);
} catch (Exception e) {
String msg = "Error when stopping container" + ", containerLauncher=" + Utils.getContainerLauncherIdentifierString(launcherId, appContext) + ", scheduler=" + Utils.getTaskSchedulerIdentifierString(event.getSchedulerId(), appContext) + ", taskCommunicator=" + Utils.getTaskCommIdentifierString(event.getTaskCommId(), appContext);
LOG.error(msg, e);
sendEvent(new DAGAppMasterEventUserServiceFatalError(DAGAppMasterEventType.CONTAINER_LAUNCHER_SERVICE_FATAL_ERROR, msg, e));
}
break;
}
}
Aggregations