Examples with TaskCommunicatorDescriptor org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor used on opensource projects

Example 6 with TaskCommunicatorDescriptor

use of org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor in project hive by apache.

the class TezSessionState method openInternal.

protected void openInternal(final HiveConf conf, Collection<String> additionalFiles, boolean isAsync, LogHelper console, Path scratchDir) throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException {
    this.conf = conf;
    // TODO Why is the queue name set again. It has already been setup via setQueueName. Do only one of the two.
    String confQueueName = conf.get(TezConfiguration.TEZ_QUEUE_NAME);
    if (queueName != null && !queueName.equals(confQueueName)) {
        LOG.warn("Resetting a queue name that was already set: was " + queueName + ", now " + confQueueName);
    }
    this.queueName = confQueueName;
    this.doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS);
    final boolean llapMode = "llap".equalsIgnoreCase(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE));
    // TODO This - at least for the session pool - will always be the hive user. How does doAs above this affect things ?
    UserGroupInformation ugi = Utils.getUGI();
    user = ugi.getShortUserName();
    LOG.info("User of session id " + sessionId + " is " + user);
    // create the tez tmp dir
    tezScratchDir = scratchDir == null ? createTezDir(sessionId) : scratchDir;
    additionalFilesNotFromConf.clear();
    if (additionalFiles != null) {
        additionalFilesNotFromConf.addAll(additionalFiles);
    }
    refreshLocalResourcesFromConf(conf);
    // unless already installed on all the cluster nodes, we'll have to
    // localize hive-exec.jar as well.
    appJarLr = createJarLocalResource(utils.getExecJarPathLocal());
    // configuration for the application master
    final Map<String, LocalResource> commonLocalResources = new HashMap<String, LocalResource>();
    commonLocalResources.put(utils.getBaseName(appJarLr), appJarLr);
    for (LocalResource lr : localizedResources) {
        commonLocalResources.put(utils.getBaseName(lr), lr);
    }
    if (llapMode) {
        // localize llap client jars
        addJarLRByClass(LlapTaskSchedulerService.class, commonLocalResources);
        addJarLRByClass(LlapProtocolClientImpl.class, commonLocalResources);
        addJarLRByClass(LlapProtocolClientProxy.class, commonLocalResources);
        addJarLRByClassName("org.apache.hadoop.registry.client.api.RegistryOperations", commonLocalResources);
    }
    // Create environment for AM.
    Map<String, String> amEnv = new HashMap<String, String>();
    MRHelpers.updateEnvBasedOnMRAMEnv(conf, amEnv);
    // and finally we're ready to create and start the session
    // generate basic tez config
    final TezConfiguration tezConfig = new TezConfiguration(conf);
    // set up the staging directory to use
    tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString());
    conf.stripHiddenConfigurations(tezConfig);
    ServicePluginsDescriptor servicePluginsDescriptor;
    Credentials llapCredentials = null;
    if (llapMode) {
        if (UserGroupInformation.isSecurityEnabled()) {
            llapCredentials = new Credentials();
            llapCredentials.addToken(LlapTokenIdentifier.KIND_NAME, getLlapToken(user, tezConfig));
        }
        // TODO Change this to not serialize the entire Configuration - minor.
        UserPayload servicePluginPayload = TezUtils.createUserPayloadFromConf(tezConfig);
        // we need plugins to handle llap and uber mode
        servicePluginsDescriptor = ServicePluginsDescriptor.create(true, new TaskSchedulerDescriptor[] { TaskSchedulerDescriptor.create(LLAP_SERVICE, LLAP_SCHEDULER).setUserPayload(servicePluginPayload) }, new ContainerLauncherDescriptor[] { ContainerLauncherDescriptor.create(LLAP_SERVICE, LLAP_LAUNCHER) }, new TaskCommunicatorDescriptor[] { TaskCommunicatorDescriptor.create(LLAP_SERVICE, LLAP_TASK_COMMUNICATOR).setUserPayload(servicePluginPayload) });
    } else {
        servicePluginsDescriptor = ServicePluginsDescriptor.create(true);
    }
    // container prewarming. tell the am how many containers we need
    if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
        int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS);
        n = Math.max(tezConfig.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT), n);
        tezConfig.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, n);
    }
    setupSessionAcls(tezConfig, conf);
    final TezClient session = TezClient.newBuilder("HIVE-" + sessionId, tezConfig).setIsSession(true).setLocalResources(commonLocalResources).setCredentials(llapCredentials).setServicePluginDescriptor(servicePluginsDescriptor).build();
    LOG.info("Opening new Tez Session (id: " + sessionId + ", scratch dir: " + tezScratchDir + ")");
    TezJobMonitor.initShutdownHook();
    if (!isAsync) {
        startSessionAndContainers(session, conf, commonLocalResources, tezConfig, false);
        this.session = session;
    } else {
        FutureTask<TezClient> sessionFuture = new FutureTask<>(new Callable<TezClient>() {

            @Override
            public TezClient call() throws Exception {
                try {
                    return startSessionAndContainers(session, conf, commonLocalResources, tezConfig, true);
                } catch (Throwable t) {
                    LOG.error("Failed to start Tez session", t);
                    throw (t instanceof Exception) ? (Exception) t : new Exception(t);
                }
            }
        });
        new Thread(sessionFuture, "Tez session start thread").start();
        // We assume here nobody will try to get session before open() returns.
        this.console = console;
        this.sessionFuture = sessionFuture;
    }
}

Example 7 with TaskCommunicatorDescriptor

use of org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor in project tez by apache.

the class TestDagTypeConverters method testServiceDescriptorTranslation.

@Test(timeout = 5000)
public void testServiceDescriptorTranslation() {
    TaskSchedulerDescriptor[] taskSchedulers;
    ContainerLauncherDescriptor[] containerLaunchers;
    TaskCommunicatorDescriptor[] taskComms;
    ServicePluginsDescriptor servicePluginsDescriptor;
    AMPluginDescriptorProto proto;
    // Uber-execution
    servicePluginsDescriptor = ServicePluginsDescriptor.create(true);
    proto = DagTypeConverters.convertServicePluginDescriptorToProto(servicePluginsDescriptor);
    assertTrue(proto.hasUberEnabled());
    assertTrue(proto.hasContainersEnabled());
    assertTrue(proto.getUberEnabled());
    assertTrue(proto.getContainersEnabled());
    assertEquals(0, proto.getTaskSchedulersCount());
    assertEquals(0, proto.getContainerLaunchersCount());
    assertEquals(0, proto.getTaskCommunicatorsCount());
    // Single plugin set specified. One with a payload.
    taskSchedulers = createTaskScheduelrs(1, false);
    containerLaunchers = createContainerLaunchers(1, false);
    taskComms = createTaskCommunicators(1, true);
    servicePluginsDescriptor = ServicePluginsDescriptor.create(taskSchedulers, containerLaunchers, taskComms);
    proto = DagTypeConverters.convertServicePluginDescriptorToProto(servicePluginsDescriptor);
    assertTrue(proto.hasUberEnabled());
    assertTrue(proto.hasContainersEnabled());
    assertFalse(proto.getUberEnabled());
    assertTrue(proto.getContainersEnabled());
    verifyPlugins(proto.getTaskSchedulersList(), 1, testScheduler, false);
    verifyPlugins(proto.getContainerLaunchersList(), 1, testLauncher, false);
    verifyPlugins(proto.getTaskCommunicatorsList(), 1, testComm, true);
    // Multiple plugin set specified. All with a payload
    taskSchedulers = createTaskScheduelrs(3, true);
    containerLaunchers = createContainerLaunchers(3, true);
    taskComms = createTaskCommunicators(3, true);
    servicePluginsDescriptor = ServicePluginsDescriptor.create(taskSchedulers, containerLaunchers, taskComms);
    proto = DagTypeConverters.convertServicePluginDescriptorToProto(servicePluginsDescriptor);
    assertTrue(proto.hasUberEnabled());
    assertTrue(proto.hasContainersEnabled());
    assertFalse(proto.getUberEnabled());
    assertTrue(proto.getContainersEnabled());
    verifyPlugins(proto.getTaskSchedulersList(), 3, testScheduler, true);
    verifyPlugins(proto.getContainerLaunchersList(), 3, testLauncher, true);
    verifyPlugins(proto.getTaskCommunicatorsList(), 3, testComm, true);
    // Single plugin set specified. One with a payload. No container execution. Uber enabled.
    taskSchedulers = createTaskScheduelrs(1, false);
    containerLaunchers = createContainerLaunchers(1, false);
    taskComms = createTaskCommunicators(1, true);
    servicePluginsDescriptor = ServicePluginsDescriptor.create(false, true, taskSchedulers, containerLaunchers, taskComms);
    proto = DagTypeConverters.convertServicePluginDescriptorToProto(servicePluginsDescriptor);
    assertTrue(proto.hasUberEnabled());
    assertTrue(proto.hasContainersEnabled());
    assertTrue(proto.getUberEnabled());
    assertFalse(proto.getContainersEnabled());
    verifyPlugins(proto.getTaskSchedulersList(), 1, testScheduler, false);
    verifyPlugins(proto.getContainerLaunchersList(), 1, testLauncher, false);
    verifyPlugins(proto.getTaskCommunicatorsList(), 1, testComm, true);
}

Example 8 with TaskCommunicatorDescriptor

use of org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor in project tez by apache.

the class TestTaskCommunicatorManager method testReportFailureFromTaskCommunicator.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskCommunicator() throws TezException {
    String dagName = DAG_NAME;
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    doReturn("testTaskCommunicator").when(appContext).getTaskCommunicatorName(0);
    doReturn(eventHandler).when(appContext).getEventHandler();
    DAG dag = mock(DAG.class);
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    doReturn(dag).when(appContext).getCurrentDAG();
    NamedEntityDescriptor<TaskCommunicatorDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>("testTaskCommunicator", TaskCommForFailureTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(namedEntityDescriptor);
    TaskCommunicatorManager taskCommManager = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), list);
    try {
        taskCommManager.init(new Configuration());
        taskCommManager.start();
        taskCommManager.registerRunningContainer(mock(ContainerId.class), 0);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains("[0:testTaskCommunicator]"));
        reset(eventHandler);
        taskCommManager.dagComplete(dag);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains("[0:testTaskCommunicator]"));
    } finally {
        taskCommManager.stop();
    }
}