Examples with NamedEntityDescriptor org.apache.tez.dag.api.NamedEntityDescriptor used on opensource projects

Example 16 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskSchedulerManager method testReportFailureFromTaskScheduler.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskScheduler() {
    String dagName = DAG_NAME;
    Configuration conf = new TezConfiguration();
    String taskSchedulerName = "testTaskScheduler";
    String expIdentifier = "[0:" + taskSchedulerName + "]";
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    doReturn(taskSchedulerName).when(appContext).getTaskSchedulerName(0);
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(conf).when(appContext).getAMConf();
    InetSocketAddress address = new InetSocketAddress("host", 55000);
    DAGClientServer dagClientServer = mock(DAGClientServer.class);
    doReturn(address).when(dagClientServer).getBindAddress();
    DAG dag = mock(DAG.class);
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    doReturn(dag).when(appContext).getCurrentDAG();
    NamedEntityDescriptor<TaskSchedulerDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>(taskSchedulerName, TaskSchedulerForFailureTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(namedEntityDescriptor);
    TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(appContext, dagClientServer, eventHandler, mock(ContainerSignatureMatcher.class), mock(WebUIService.class), list, false, new HadoopShimsLoader(appContext.getAMConf()).getHadoopShim()) {

        @Override
        TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
            // Avoid wrapping in threads
            return rawContext;
        }
    };
    try {
        taskSchedulerManager.init(new TezConfiguration());
        taskSchedulerManager.start();
        taskSchedulerManager.getTotalResources(0);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains(expIdentifier));
        reset(eventHandler);
        taskSchedulerManager.getAvailableResources(0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains(expIdentifier));
    } finally {
        taskSchedulerManager.stop();
    }
}

Example 17 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestContainerLauncherManager method testCustomLauncherSpecified.

@Test(timeout = 5000)
public void testCustomLauncherSpecified() throws IOException, TezException {
    Configuration conf = new Configuration(false);
    AppContext appContext = mock(AppContext.class);
    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
    String customLauncherName = "customLauncher";
    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload customPayload = UserPayload.create(bb);
    launcherDescriptors.add(new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName()).setUserPayload(customPayload));
    ContainerLaucherRouterForMultipleLauncherTest clr = new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, launcherDescriptors, true);
    try {
        clr.init(conf);
        clr.start();
        assertEquals(1, clr.getNumContainerLaunchers());
        assertFalse(clr.getYarnContainerLauncherCreated());
        assertFalse(clr.getUberContainerLauncherCreated());
        assertEquals(customLauncherName, clr.getContainerLauncherName(0));
        assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
    } finally {
        clr.stop();
    }
}

Example 18 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestContainerLauncherManager method testMultipleContainerLaunchers.

@Test(timeout = 5000)
public void testMultipleContainerLaunchers() throws IOException, TezException {
    Configuration conf = new Configuration(false);
    conf.set("testkey", "testvalue");
    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
    AppContext appContext = mock(AppContext.class);
    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
    String customLauncherName = "customLauncher";
    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload customPayload = UserPayload.create(bb);
    launcherDescriptors.add(new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName()).setUserPayload(customPayload));
    launcherDescriptors.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload));
    ContainerLaucherRouterForMultipleLauncherTest clr = new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, launcherDescriptors, true);
    try {
        clr.init(conf);
        clr.start();
        assertEquals(2, clr.getNumContainerLaunchers());
        assertTrue(clr.getYarnContainerLauncherCreated());
        assertFalse(clr.getUberContainerLauncherCreated());
        assertEquals(customLauncherName, clr.getContainerLauncherName(0));
        assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
        assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
        Configuration confParsed = TezUtils.createConfFromUserPayload(clr.getContainerLauncherContext(1).getInitialUserPayload());
        assertEquals("testvalue", confParsed.get("testkey"));
    } finally {
        clr.stop();
    }
}

Example 19 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskCommunicatorManager1 method setUp.

@Before
public void setUp() throws TezException {
    appId = ApplicationId.newInstance(1000, 1);
    appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    dag = mock(DAG.class);
    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
    vertexID = TezVertexID.getInstance(dagID, 1);
    taskID = TezTaskID.getInstance(vertexID, 1);
    taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
    credentials = new Credentials();
    amContainerMap = mock(AMContainerMap.class);
    Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
    eventHandler = mock(EventHandler.class);
    MockClock clock = new MockClock();
    appContext = mock(AppContext.class);
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(dag).when(appContext).getCurrentDAG();
    doReturn(appAcls).when(appContext).getApplicationACLs();
    doReturn(amContainerMap).when(appContext).getAllContainers();
    doReturn(clock).when(appContext).getClock();
    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
    doReturn(credentials).when(appContext).getAppCredentials();
    NodeId nodeId = NodeId.newInstance("localhost", 0);
    AMContainer amContainer = mock(AMContainer.class);
    Container container = mock(Container.class);
    doReturn(nodeId).when(container).getNodeId();
    doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
    doReturn(container).when(amContainer).getContainer();
    Configuration conf = new TezConfiguration();
    UserPayload defaultPayload;
    try {
        defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    } catch (IOException e) {
        throw new TezUncheckedException(e);
    }
    taskAttemptListener = new TaskCommunicatorManagerInterfaceImplForTest(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(defaultPayload)));
    taskSpec = mock(TaskSpec.class);
    doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();
    amContainerTask = new AMContainerTask(taskSpec, null, null, false, 0);
    containerTask = null;
}

Example 20 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskCommunicatorManager1 method testPortRange.

private boolean testPortRange(int port) {
    boolean succeedToAllocate = true;
    try {
        Configuration conf = new Configuration();
        JobTokenIdentifier identifier = new JobTokenIdentifier(new Text("fakeIdentifier"));
        Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, new JobTokenSecretManager());
        sessionToken.setService(identifier.getJobId());
        TokenCache.setSessionToken(sessionToken, credentials);
        conf.set(TezConfiguration.TEZ_AM_TASK_AM_PORT_RANGE, port + "-" + port);
        UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
        taskAttemptListener = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload)));
        taskAttemptListener.init(conf);
        taskAttemptListener.start();
        int resultedPort = taskAttemptListener.getTaskCommunicator(0).getAddress().getPort();
        assertEquals(port, resultedPort);
    } catch (Exception e) {
        succeedToAllocate = false;
    } finally {
        if (taskAttemptListener != null) {
            try {
                taskAttemptListener.close();
            } catch (IOException e) {
                e.printStackTrace();
                fail("fail to stop TaskAttemptListener");
            }
        }
    }
    return succeedToAllocate;
}