Search in sources :

Example 26 with TaskSchedulerContext

use of org.apache.tez.serviceplugins.api.TaskSchedulerContext in project tez by apache.

the class TestTaskSchedulerManager method testReportFailureFromTaskScheduler.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskScheduler() {
    String dagName = DAG_NAME;
    Configuration conf = new TezConfiguration();
    String taskSchedulerName = "testTaskScheduler";
    String expIdentifier = "[0:" + taskSchedulerName + "]";
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    doReturn(taskSchedulerName).when(appContext).getTaskSchedulerName(0);
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(conf).when(appContext).getAMConf();
    InetSocketAddress address = new InetSocketAddress("host", 55000);
    DAGClientServer dagClientServer = mock(DAGClientServer.class);
    doReturn(address).when(dagClientServer).getBindAddress();
    DAG dag = mock(DAG.class);
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    doReturn(dag).when(appContext).getCurrentDAG();
    NamedEntityDescriptor<TaskSchedulerDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>(taskSchedulerName, TaskSchedulerForFailureTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(namedEntityDescriptor);
    TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(appContext, dagClientServer, eventHandler, mock(ContainerSignatureMatcher.class), mock(WebUIService.class), list, false, new HadoopShimsLoader(appContext.getAMConf()).getHadoopShim()) {

        @Override
        TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
            // Avoid wrapping in threads
            return rawContext;
        }
    };
    try {
        taskSchedulerManager.init(new TezConfiguration());
        taskSchedulerManager.start();
        taskSchedulerManager.getTotalResources(0);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains(expIdentifier));
        reset(eventHandler);
        taskSchedulerManager.getAvailableResources(0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains(expIdentifier));
    } finally {
        taskSchedulerManager.stop();
    }
}
Also used : DAGAppMasterEventUserServiceFatalError(org.apache.tez.dag.app.dag.event.DAGAppMasterEventUserServiceFatalError) Configuration(org.apache.hadoop.conf.Configuration) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) TaskSchedulerDescriptor(org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor) TaskSchedulerContext(org.apache.tez.serviceplugins.api.TaskSchedulerContext) InetSocketAddress(java.net.InetSocketAddress) AppContext(org.apache.tez.dag.app.AppContext) ContainerSignatureMatcher(org.apache.tez.common.ContainerSignatureMatcher) EventHandler(org.apache.hadoop.yarn.event.EventHandler) DAG(org.apache.tez.dag.app.dag.DAG) DAGEventTerminateDag(org.apache.tez.dag.app.dag.event.DAGEventTerminateDag) NamedEntityDescriptor(org.apache.tez.dag.api.NamedEntityDescriptor) LinkedList(java.util.LinkedList) WebUIService(org.apache.tez.dag.app.web.WebUIService) HadoopShimsLoader(org.apache.tez.hadoop.shim.HadoopShimsLoader) TezDAGID(org.apache.tez.dag.records.TezDAGID) Event(org.apache.hadoop.yarn.event.Event) DAGClientServer(org.apache.tez.dag.api.client.DAGClientServer) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) DagInfoImplForTest(org.apache.tez.dag.helpers.DagInfoImplForTest) Test(org.junit.Test)

Example 27 with TaskSchedulerContext

use of org.apache.tez.serviceplugins.api.TaskSchedulerContext in project tez by apache.

the class DagAwareYarnTaskScheduler method start.

@Override
public void start() throws Exception {
    super.start();
    client.start();
    if (shouldReuseContainers) {
        reuseExecutor = createExecutor();
    }
    TaskSchedulerContext ctx = getContext();
    RegisterApplicationMasterResponse response = client.registerApplicationMaster(ctx.getAppHostName(), ctx.getAppClientPort(), ctx.getAppTrackingUrl());
    ctx.setApplicationRegistrationData(response.getMaximumResourceCapability(), response.getApplicationACLs(), response.getClientToAMTokenMasterKey(), response.getQueue());
    if (response.getSchedulerResourceTypes().contains(SchedulerResourceTypes.CPU)) {
        resourceCalculator = new MemCpuResourceCalculator();
    } else {
        resourceCalculator = new MemResourceCalculator();
    }
}
Also used : TaskSchedulerContext(org.apache.tez.serviceplugins.api.TaskSchedulerContext) RegisterApplicationMasterResponse(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse)

Aggregations

TaskSchedulerContext (org.apache.tez.serviceplugins.api.TaskSchedulerContext)27 TezConfiguration (org.apache.tez.dag.api.TezConfiguration)24 Test (org.junit.Test)24 Configuration (org.apache.hadoop.conf.Configuration)23 TaskSchedulerContextDrainable (org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.TaskSchedulerContextDrainable)19 TestTaskSchedulerHelpers.setupMockTaskSchedulerContext (org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.setupMockTaskSchedulerContext)19 Container (org.apache.hadoop.yarn.api.records.Container)18 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)18 Priority (org.apache.hadoop.yarn.api.records.Priority)18 AppFinalStatus (org.apache.tez.serviceplugins.api.TaskSchedulerContext.AppFinalStatus)16 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)15 RegisterApplicationMasterResponse (org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse)12 DagInfo (org.apache.tez.serviceplugins.api.DagInfo)12 NodeId (org.apache.hadoop.yarn.api.records.NodeId)11 BitSet (java.util.BitSet)10 Resource (org.apache.hadoop.yarn.api.records.Resource)10 AMRMClientAsyncForTest (org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.AMRMClientAsyncForTest)10 AMRMClientForTest (org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.AMRMClientForTest)10 TaskSchedulerWithDrainableContext (org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.TaskSchedulerWithDrainableContext)10 CookieContainerRequest (org.apache.tez.dag.app.rm.YarnTaskSchedulerService.CookieContainerRequest)10