Examples with NamedEntityDescriptor org.apache.tez.dag.api.NamedEntityDescriptor used on opensource projects

Example 6 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskSchedulerManager method testShutdownBeforeStartTaskScheduler.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testShutdownBeforeStartTaskScheduler() {
    Configuration conf = new TezConfiguration();
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    doReturn(conf).when(appContext).getAMConf();
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(null);
    TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(appContext, null, null, null, null, list, false, null);
    assertFalse("Should not return true unless actually unregistered successfully", taskSchedulerManager.hasUnregistered());
}

Example 7 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskCommunicatorManager1 method testPortRange_NotSpecified.

// TODO TEZ-2003 Move this into TestTezTaskCommunicator. Potentially other tests as well.
@Test(timeout = 5000)
public void testPortRange_NotSpecified() throws IOException, TezException {
    Configuration conf = new Configuration();
    JobTokenIdentifier identifier = new JobTokenIdentifier(new Text("fakeIdentifier"));
    Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, new JobTokenSecretManager());
    sessionToken.setService(identifier.getJobId());
    TokenCache.setSessionToken(sessionToken, credentials);
    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
    taskAttemptListener = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload)));
    // no exception happen, should started properly
    taskAttemptListener.init(conf);
    taskAttemptListener.start();
}

Example 8 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class DAGAppMaster method parsePlugin.

@VisibleForTesting
public static void parsePlugin(List<NamedEntityDescriptor> resultList, BiMap<String, Integer> pluginMap, List<TezNamedEntityDescriptorProto> namedEntityDescriptorProtos, boolean tezYarnEnabled, boolean uberEnabled, UserPayload defaultPayload) {
    if (tezYarnEnabled) {
        // Default classnames will be populated by individual components
        NamedEntityDescriptor r = new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(defaultPayload);
        addDescriptor(resultList, pluginMap, r);
    }
    if (uberEnabled) {
        // Default classnames will be populated by individual components
        NamedEntityDescriptor r = new NamedEntityDescriptor(TezConstants.getTezUberServicePluginName(), null).setUserPayload(defaultPayload);
        addDescriptor(resultList, pluginMap, r);
    }
    if (namedEntityDescriptorProtos != null) {
        for (TezNamedEntityDescriptorProto namedEntityDescriptorProto : namedEntityDescriptorProtos) {
            NamedEntityDescriptor namedEntityDescriptor = DagTypeConverters.convertNamedDescriptorFromProto(namedEntityDescriptorProto);
            addDescriptor(resultList, pluginMap, namedEntityDescriptor);
        }
    }
}

Example 9 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class DAGAppMaster method serviceInit.

@Override
public synchronized void serviceInit(final Configuration conf) throws Exception {
    this.amConf = conf;
    initResourceCalculatorPlugins();
    this.hadoopShim = new HadoopShimsLoader(this.amConf).getHadoopShim();
    long sleepTimeBeforeSecs = this.amConf.getLong(TezConfiguration.TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS, TezConstants.TEZ_DAG_SLEEP_TIME_BEFORE_EXIT);
    if (sleepTimeBeforeSecs >= 0) {
        this.shutdownHandler.setSleepTimeBeforeExit(sleepTimeBeforeSecs);
    }
    this.isLocal = conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(amConf);
    List<NamedEntityDescriptor> taskSchedulerDescriptors = Lists.newLinkedList();
    List<NamedEntityDescriptor> containerLauncherDescriptors = Lists.newLinkedList();
    List<NamedEntityDescriptor> taskCommunicatorDescriptors = Lists.newLinkedList();
    parseAllPlugins(taskSchedulerDescriptors, taskSchedulers, containerLauncherDescriptors, containerLaunchers, taskCommunicatorDescriptors, taskCommunicators, amPluginDescriptorProto, isLocal, defaultPayload);
    LOG.info(buildPluginComponentLog(taskSchedulerDescriptors, taskSchedulers, "TaskSchedulers"));
    LOG.info(buildPluginComponentLog(containerLauncherDescriptors, containerLaunchers, "ContainerLaunchers"));
    LOG.info(buildPluginComponentLog(taskCommunicatorDescriptors, taskCommunicators, "TaskCommunicators"));
    boolean disableVersionCheck = conf.getBoolean(TezConfiguration.TEZ_AM_DISABLE_CLIENT_VERSION_CHECK, TezConfiguration.TEZ_AM_DISABLE_CLIENT_VERSION_CHECK_DEFAULT);
    // Check client - AM version compatibility
    LOG.info("Comparing client version with AM version" + ", clientVersion=" + clientVersion + ", AMVersion=" + dagVersionInfo.getVersion());
    Simple2LevelVersionComparator versionComparator = new Simple2LevelVersionComparator();
    if (versionComparator.compare(clientVersion, dagVersionInfo.getVersion()) != 0) {
        versionMismatchDiagnostics = "Incompatible versions found" + ", clientVersion=" + clientVersion + ", AMVersion=" + dagVersionInfo.getVersion();
        addDiagnostic(versionMismatchDiagnostics);
        if (disableVersionCheck) {
            LOG.warn("Ignoring client-AM version mismatch as check disabled. " + versionMismatchDiagnostics);
        } else {
            LOG.error(versionMismatchDiagnostics);
            versionMismatch = true;
        }
    }
    dispatcher = createDispatcher();
    if (isLocal) {
        conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
        conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS_DEFAULT);
    } else {
        dispatcher.enableExitOnDispatchException();
    }
    String strAppId = this.appAttemptID.getApplicationId().toString();
    this.tezSystemStagingDir = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
    context = new RunningAppContext(conf);
    this.aclManager = new ACLManager(appMasterUgi.getShortUserName(), this.amConf);
    clientHandler = new DAGClientHandler(this);
    addIfService(dispatcher, false);
    recoveryDataDir = TezCommonUtils.getRecoveryPath(tezSystemStagingDir, conf);
    recoveryFS = recoveryDataDir.getFileSystem(conf);
    currentRecoveryDataDir = TezCommonUtils.getAttemptRecoveryPath(recoveryDataDir, appAttemptID.getAttemptId());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Stage directory information for AppAttemptId :" + this.appAttemptID + " tezSystemStagingDir :" + tezSystemStagingDir + " recoveryDataDir :" + recoveryDataDir + " recoveryAttemptDir :" + currentRecoveryDataDir);
    }
    recoveryEnabled = conf.getBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT);
    clientRpcServer = new DAGClientServer(clientHandler, appAttemptID, recoveryFS);
    addIfService(clientRpcServer, true);
    taskHeartbeatHandler = createTaskHeartbeatHandler(context, conf);
    addIfService(taskHeartbeatHandler, true);
    containerHeartbeatHandler = createContainerHeartbeatHandler(context, conf);
    addIfService(containerHeartbeatHandler, true);
    sessionToken = TokenCache.getSessionToken(amCredentials);
    if (sessionToken == null) {
        throw new RuntimeException("Could not find session token in AM Credentials");
    }
    // Prepare the TaskAttemptListener server for authentication of Containers
    // TaskAttemptListener gets the information via jobTokenSecretManager.
    jobTokenSecretManager.addTokenForJob(appAttemptID.getApplicationId().toString(), sessionToken);
    // service to handle requests to TaskUmbilicalProtocol
    taskCommunicatorManager = createTaskCommunicatorManager(context, taskHeartbeatHandler, containerHeartbeatHandler, taskCommunicatorDescriptors);
    addIfService(taskCommunicatorManager, true);
    containerSignatureMatcher = createContainerSignatureMatcher();
    containers = new AMContainerMap(containerHeartbeatHandler, taskCommunicatorManager, containerSignatureMatcher, context);
    addIfService(containers, true);
    dispatcher.register(AMContainerEventType.class, containers);
    nodes = new AMNodeTracker(dispatcher.getEventHandler(), context);
    addIfService(nodes, true);
    dispatcher.register(AMNodeEventType.class, nodes);
    this.dagEventDispatcher = new DagEventDispatcher();
    this.vertexEventDispatcher = new VertexEventDispatcher();
    // register the event dispatchers
    dispatcher.register(DAGAppMasterEventType.class, new DAGAppMasterEventHandler());
    dispatcher.register(DAGEventType.class, dagEventDispatcher);
    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
    boolean useConcurrentDispatcher = conf.getBoolean(TezConfiguration.TEZ_AM_USE_CONCURRENT_DISPATCHER, TezConfiguration.TEZ_AM_USE_CONCURRENT_DISPATCHER_DEFAULT);
    LOG.info("Using concurrent dispatcher: " + useConcurrentDispatcher);
    if (!useConcurrentDispatcher) {
        dispatcher.register(TaskEventType.class, new TaskEventDispatcher());
        dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventDispatcher());
    } else {
        int concurrency = conf.getInt(TezConfiguration.TEZ_AM_CONCURRENT_DISPATCHER_CONCURRENCY, TezConfiguration.TEZ_AM_CONCURRENT_DISPATCHER_CONCURRENCY_DEFAULT);
        AsyncDispatcherConcurrent sharedDispatcher = dispatcher.registerAndCreateDispatcher(TaskEventType.class, new TaskEventDispatcher(), "TaskAndAttemptEventThread", concurrency);
        dispatcher.registerWithExistingDispatcher(TaskAttemptEventType.class, new TaskAttemptEventDispatcher(), sharedDispatcher);
    }
    // register other delegating dispatchers
    dispatcher.registerAndCreateDispatcher(SpeculatorEventType.class, new SpeculatorEventHandler(), "Speculator");
    if (enableWebUIService()) {
        this.webUIService = new WebUIService(context);
        addIfService(webUIService, false);
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Web UI Service is not enabled.");
        }
    }
    this.taskSchedulerManager = createTaskSchedulerManager(taskSchedulerDescriptors);
    addIfService(taskSchedulerManager, true);
    if (enableWebUIService()) {
        addIfServiceDependency(taskSchedulerManager, webUIService);
    }
    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
    addIfServiceDependency(taskSchedulerManager, clientRpcServer);
    this.containerLauncherManager = createContainerLauncherManager(containerLauncherDescriptors, isLocal);
    addIfService(containerLauncherManager, true);
    dispatcher.register(ContainerLauncherEventType.class, containerLauncherManager);
    historyEventHandler = createHistoryEventHandler(context);
    addIfService(historyEventHandler, true);
    this.sessionTimeoutInterval = TezCommonUtils.getDAGSessionTimeout(amConf);
    this.clientAMHeartbeatTimeoutIntervalMillis = TezCommonUtils.getAMClientHeartBeatTimeoutMillis(amConf);
    if (!versionMismatch) {
        if (isSession) {
            FileInputStream sessionResourcesStream = null;
            try {
                sessionResourcesStream = new FileInputStream(new File(workingDirectory, TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
                PlanLocalResourcesProto amLocalResourceProto = PlanLocalResourcesProto.parseDelimitedFrom(sessionResourcesStream);
                amResources.putAll(DagTypeConverters.convertFromPlanLocalResources(amLocalResourceProto));
            } finally {
                if (sessionResourcesStream != null) {
                    sessionResourcesStream.close();
                }
            }
        }
    }
    rawExecutor = Executors.newCachedThreadPool(new ThreadFactoryBuilder().setDaemon(true).setNameFormat("App Shared Pool - " + "#%d").build());
    execService = MoreExecutors.listeningDecorator(rawExecutor);
    initServices(conf);
    super.serviceInit(conf);
    if (!versionMismatch) {
        if (this.appAttemptID.getAttemptId() == 1) {
            AppLaunchedEvent appLaunchedEvent = new AppLaunchedEvent(appAttemptID.getApplicationId(), startTime, appSubmitTime, appMasterUgi.getShortUserName(), this.amConf, dagVersionInfo);
            historyEventHandler.handle(new DAGHistoryEvent(appLaunchedEvent));
        }
        AMLaunchedEvent launchedEvent = new AMLaunchedEvent(appAttemptID, startTime, appSubmitTime, appMasterUgi.getShortUserName());
        historyEventHandler.handle(new DAGHistoryEvent(launchedEvent));
        this.state = DAGAppMasterState.INITED;
    } else {
        this.state = DAGAppMasterState.ERROR;
    }
}

Example 10 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestDAGAppMaster method testPluginParsing.

@Test(timeout = 5000)
public void testPluginParsing() throws IOException {
    BiMap<String, Integer> pluginMap = HashBiMap.create();
    Configuration conf = new Configuration(false);
    conf.set("testkey", "testval");
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    List<TezNamedEntityDescriptorProto> entityDescriptors = new LinkedList<>();
    List<NamedEntityDescriptor> entities;
    // Test empty descriptor list, yarn enabled
    pluginMap.clear();
    entities = new LinkedList<>();
    DAGAppMaster.parsePlugin(entities, pluginMap, null, true, false, defaultPayload);
    assertEquals(1, pluginMap.size());
    assertEquals(1, entities.size());
    assertTrue(pluginMap.containsKey(TezConstants.getTezYarnServicePluginName()));
    assertTrue(0 == pluginMap.get(TezConstants.getTezYarnServicePluginName()));
    assertEquals("testval", TezUtils.createConfFromUserPayload(entities.get(0).getUserPayload()).get("testkey"));
    // Test empty descriptor list, uber enabled
    pluginMap.clear();
    entities = new LinkedList<>();
    DAGAppMaster.parsePlugin(entities, pluginMap, null, false, true, defaultPayload);
    assertEquals(1, pluginMap.size());
    assertEquals(1, entities.size());
    assertTrue(pluginMap.containsKey(TezConstants.getTezUberServicePluginName()));
    assertTrue(0 == pluginMap.get(TezConstants.getTezUberServicePluginName()));
    assertEquals("testval", TezUtils.createConfFromUserPayload(entities.get(0).getUserPayload()).get("testkey"));
    // Test empty descriptor list, yarn enabled, uber enabled
    pluginMap.clear();
    entities = new LinkedList<>();
    DAGAppMaster.parsePlugin(entities, pluginMap, null, true, true, defaultPayload);
    assertEquals(2, pluginMap.size());
    assertEquals(2, entities.size());
    assertTrue(pluginMap.containsKey(TezConstants.getTezYarnServicePluginName()));
    assertTrue(0 == pluginMap.get(TezConstants.getTezYarnServicePluginName()));
    assertTrue(pluginMap.containsKey(TezConstants.getTezUberServicePluginName()));
    assertTrue(1 == pluginMap.get(TezConstants.getTezUberServicePluginName()));
    String pluginName = "d1";
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    TezNamedEntityDescriptorProto d1 = TezNamedEntityDescriptorProto.newBuilder().setName(pluginName).setEntityDescriptor(DAGProtos.TezEntityDescriptorProto.newBuilder().setClassName("d1Class").setTezUserPayload(TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(bb)))).build();
    entityDescriptors.add(d1);
    // Test descriptor, no yarn, no uber
    pluginMap.clear();
    entities = new LinkedList<>();
    DAGAppMaster.parsePlugin(entities, pluginMap, entityDescriptors, false, false, defaultPayload);
    assertEquals(1, pluginMap.size());
    assertEquals(1, entities.size());
    assertTrue(pluginMap.containsKey(pluginName));
    assertTrue(0 == pluginMap.get(pluginName));
    // Test descriptor, yarn and uber
    pluginMap.clear();
    entities = new LinkedList<>();
    DAGAppMaster.parsePlugin(entities, pluginMap, entityDescriptors, true, true, defaultPayload);
    assertEquals(3, pluginMap.size());
    assertEquals(3, entities.size());
    assertTrue(pluginMap.containsKey(TezConstants.getTezYarnServicePluginName()));
    assertTrue(0 == pluginMap.get(TezConstants.getTezYarnServicePluginName()));
    assertTrue(pluginMap.containsKey(TezConstants.getTezUberServicePluginName()));
    assertTrue(1 == pluginMap.get(TezConstants.getTezUberServicePluginName()));
    assertTrue(pluginMap.containsKey(pluginName));
    assertTrue(2 == pluginMap.get(pluginName));
    entityDescriptors.clear();
}