Examples with DAGClientHandler org.apache.tez.dag.api.client.DAGClientHandler used on opensource projects

Example 1 with DAGClientHandler

use of org.apache.tez.dag.api.client.DAGClientHandler in project tez by apache.

the class DAGAppMaster method serviceInit.

@Override
public synchronized void serviceInit(final Configuration conf) throws Exception {
    this.amConf = conf;
    initResourceCalculatorPlugins();
    this.hadoopShim = new HadoopShimsLoader(this.amConf).getHadoopShim();
    long sleepTimeBeforeSecs = this.amConf.getLong(TezConfiguration.TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS, TezConstants.TEZ_DAG_SLEEP_TIME_BEFORE_EXIT);
    if (sleepTimeBeforeSecs >= 0) {
        this.shutdownHandler.setSleepTimeBeforeExit(sleepTimeBeforeSecs);
    }
    this.isLocal = conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(amConf);
    List<NamedEntityDescriptor> taskSchedulerDescriptors = Lists.newLinkedList();
    List<NamedEntityDescriptor> containerLauncherDescriptors = Lists.newLinkedList();
    List<NamedEntityDescriptor> taskCommunicatorDescriptors = Lists.newLinkedList();
    parseAllPlugins(taskSchedulerDescriptors, taskSchedulers, containerLauncherDescriptors, containerLaunchers, taskCommunicatorDescriptors, taskCommunicators, amPluginDescriptorProto, isLocal, defaultPayload);
    LOG.info(buildPluginComponentLog(taskSchedulerDescriptors, taskSchedulers, "TaskSchedulers"));
    LOG.info(buildPluginComponentLog(containerLauncherDescriptors, containerLaunchers, "ContainerLaunchers"));
    LOG.info(buildPluginComponentLog(taskCommunicatorDescriptors, taskCommunicators, "TaskCommunicators"));
    boolean disableVersionCheck = conf.getBoolean(TezConfiguration.TEZ_AM_DISABLE_CLIENT_VERSION_CHECK, TezConfiguration.TEZ_AM_DISABLE_CLIENT_VERSION_CHECK_DEFAULT);
    // Check client - AM version compatibility
    LOG.info("Comparing client version with AM version" + ", clientVersion=" + clientVersion + ", AMVersion=" + dagVersionInfo.getVersion());
    Simple2LevelVersionComparator versionComparator = new Simple2LevelVersionComparator();
    if (versionComparator.compare(clientVersion, dagVersionInfo.getVersion()) != 0) {
        versionMismatchDiagnostics = "Incompatible versions found" + ", clientVersion=" + clientVersion + ", AMVersion=" + dagVersionInfo.getVersion();
        addDiagnostic(versionMismatchDiagnostics);
        if (disableVersionCheck) {
            LOG.warn("Ignoring client-AM version mismatch as check disabled. " + versionMismatchDiagnostics);
        } else {
            LOG.error(versionMismatchDiagnostics);
            versionMismatch = true;
        }
    }
    dispatcher = createDispatcher();
    if (isLocal) {
        conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
        conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS_DEFAULT);
    } else {
        dispatcher.enableExitOnDispatchException();
    }
    String strAppId = this.appAttemptID.getApplicationId().toString();
    this.tezSystemStagingDir = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
    context = new RunningAppContext(conf);
    this.aclManager = new ACLManager(appMasterUgi.getShortUserName(), this.amConf);
    clientHandler = new DAGClientHandler(this);
    addIfService(dispatcher, false);
    recoveryDataDir = TezCommonUtils.getRecoveryPath(tezSystemStagingDir, conf);
    recoveryFS = recoveryDataDir.getFileSystem(conf);
    currentRecoveryDataDir = TezCommonUtils.getAttemptRecoveryPath(recoveryDataDir, appAttemptID.getAttemptId());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Stage directory information for AppAttemptId :" + this.appAttemptID + " tezSystemStagingDir :" + tezSystemStagingDir + " recoveryDataDir :" + recoveryDataDir + " recoveryAttemptDir :" + currentRecoveryDataDir);
    }
    recoveryEnabled = conf.getBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT);
    clientRpcServer = new DAGClientServer(clientHandler, appAttemptID, recoveryFS);
    addIfService(clientRpcServer, true);
    taskHeartbeatHandler = createTaskHeartbeatHandler(context, conf);
    addIfService(taskHeartbeatHandler, true);
    containerHeartbeatHandler = createContainerHeartbeatHandler(context, conf);
    addIfService(containerHeartbeatHandler, true);
    sessionToken = TokenCache.getSessionToken(amCredentials);
    if (sessionToken == null) {
        throw new RuntimeException("Could not find session token in AM Credentials");
    }
    // Prepare the TaskAttemptListener server for authentication of Containers
    // TaskAttemptListener gets the information via jobTokenSecretManager.
    jobTokenSecretManager.addTokenForJob(appAttemptID.getApplicationId().toString(), sessionToken);
    // service to handle requests to TaskUmbilicalProtocol
    taskCommunicatorManager = createTaskCommunicatorManager(context, taskHeartbeatHandler, containerHeartbeatHandler, taskCommunicatorDescriptors);
    addIfService(taskCommunicatorManager, true);
    containerSignatureMatcher = createContainerSignatureMatcher();
    containers = new AMContainerMap(containerHeartbeatHandler, taskCommunicatorManager, containerSignatureMatcher, context);
    addIfService(containers, true);
    dispatcher.register(AMContainerEventType.class, containers);
    nodes = new AMNodeTracker(dispatcher.getEventHandler(), context);
    addIfService(nodes, true);
    dispatcher.register(AMNodeEventType.class, nodes);
    this.dagEventDispatcher = new DagEventDispatcher();
    this.vertexEventDispatcher = new VertexEventDispatcher();
    // register the event dispatchers
    dispatcher.register(DAGAppMasterEventType.class, new DAGAppMasterEventHandler());
    dispatcher.register(DAGEventType.class, dagEventDispatcher);
    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
    boolean useConcurrentDispatcher = conf.getBoolean(TezConfiguration.TEZ_AM_USE_CONCURRENT_DISPATCHER, TezConfiguration.TEZ_AM_USE_CONCURRENT_DISPATCHER_DEFAULT);
    LOG.info("Using concurrent dispatcher: " + useConcurrentDispatcher);
    if (!useConcurrentDispatcher) {
        dispatcher.register(TaskEventType.class, new TaskEventDispatcher());
        dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventDispatcher());
    } else {
        int concurrency = conf.getInt(TezConfiguration.TEZ_AM_CONCURRENT_DISPATCHER_CONCURRENCY, TezConfiguration.TEZ_AM_CONCURRENT_DISPATCHER_CONCURRENCY_DEFAULT);
        AsyncDispatcherConcurrent sharedDispatcher = dispatcher.registerAndCreateDispatcher(TaskEventType.class, new TaskEventDispatcher(), "TaskAndAttemptEventThread", concurrency);
        dispatcher.registerWithExistingDispatcher(TaskAttemptEventType.class, new TaskAttemptEventDispatcher(), sharedDispatcher);
    }
    // register other delegating dispatchers
    dispatcher.registerAndCreateDispatcher(SpeculatorEventType.class, new SpeculatorEventHandler(), "Speculator");
    if (enableWebUIService()) {
        this.webUIService = new WebUIService(context);
        addIfService(webUIService, false);
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Web UI Service is not enabled.");
        }
    }
    this.taskSchedulerManager = createTaskSchedulerManager(taskSchedulerDescriptors);
    addIfService(taskSchedulerManager, true);
    if (enableWebUIService()) {
        addIfServiceDependency(taskSchedulerManager, webUIService);
    }
    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
    addIfServiceDependency(taskSchedulerManager, clientRpcServer);
    this.containerLauncherManager = createContainerLauncherManager(containerLauncherDescriptors, isLocal);
    addIfService(containerLauncherManager, true);
    dispatcher.register(ContainerLauncherEventType.class, containerLauncherManager);
    historyEventHandler = createHistoryEventHandler(context);
    addIfService(historyEventHandler, true);
    this.sessionTimeoutInterval = TezCommonUtils.getDAGSessionTimeout(amConf);
    this.clientAMHeartbeatTimeoutIntervalMillis = TezCommonUtils.getAMClientHeartBeatTimeoutMillis(amConf);
    if (!versionMismatch) {
        if (isSession) {
            FileInputStream sessionResourcesStream = null;
            try {
                sessionResourcesStream = new FileInputStream(new File(workingDirectory, TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
                PlanLocalResourcesProto amLocalResourceProto = PlanLocalResourcesProto.parseDelimitedFrom(sessionResourcesStream);
                amResources.putAll(DagTypeConverters.convertFromPlanLocalResources(amLocalResourceProto));
            } finally {
                if (sessionResourcesStream != null) {
                    sessionResourcesStream.close();
                }
            }
        }
    }
    rawExecutor = Executors.newCachedThreadPool(new ThreadFactoryBuilder().setDaemon(true).setNameFormat("App Shared Pool - " + "#%d").build());
    execService = MoreExecutors.listeningDecorator(rawExecutor);
    initServices(conf);
    super.serviceInit(conf);
    if (!versionMismatch) {
        if (this.appAttemptID.getAttemptId() == 1) {
            AppLaunchedEvent appLaunchedEvent = new AppLaunchedEvent(appAttemptID.getApplicationId(), startTime, appSubmitTime, appMasterUgi.getShortUserName(), this.amConf, dagVersionInfo);
            historyEventHandler.handle(new DAGHistoryEvent(appLaunchedEvent));
        }
        AMLaunchedEvent launchedEvent = new AMLaunchedEvent(appAttemptID, startTime, appSubmitTime, appMasterUgi.getShortUserName());
        historyEventHandler.handle(new DAGHistoryEvent(launchedEvent));
        this.state = DAGAppMasterState.INITED;
    } else {
        this.state = DAGAppMasterState.ERROR;
    }
}

Example 2 with DAGClientHandler

use of org.apache.tez.dag.api.client.DAGClientHandler in project tez by apache.

the class TestDAGClientAMProtocolBlockingPBServerImpl method testSubmitDagInSessionWithLargeDagPlan.

@Test(timeout = 100000)
@SuppressWarnings("unchecked")
public void testSubmitDagInSessionWithLargeDagPlan() throws Exception {
    int maxIPCMsgSize = 1024;
    String dagPlanName = "dagplan-name";
    File requestFile = tmpFolder.newFile("request-file");
    TezConfiguration conf = new TezConfiguration();
    conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize);
    // Check with 70 MB (64 MB is CodedInputStream's default limit in earlier versions of protobuf)
    byte[] randomBytes = new byte[70 << 20];
    (new Random()).nextBytes(randomBytes);
    UserPayload payload = UserPayload.create(ByteBuffer.wrap(randomBytes));
    Vertex vertex = Vertex.create("V", ProcessorDescriptor.create("P").setUserPayload(payload), 1);
    DAGPlan dagPlan = DAG.create(dagPlanName).addVertex(vertex).createDag(conf, null, null, null, false);
    String lrName = "localResource";
    String scheme = "file";
    String host = "localhost";
    int port = 80;
    String path = "/test";
    URL lrURL = URL.newInstance(scheme, host, port, path);
    LocalResource localResource = LocalResource.newInstance(lrURL, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1);
    Map<String, LocalResource> localResources = new HashMap<>();
    localResources.put(lrName, localResource);
    SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan).setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(localResources));
    try (FileOutputStream fileOutputStream = new FileOutputStream(requestFile)) {
        requestBuilder.build().writeTo(fileOutputStream);
    }
    DAGClientHandler dagClientHandler = mock(DAGClientHandler.class);
    ACLManager aclManager = mock(ACLManager.class);
    DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy(new DAGClientAMProtocolBlockingPBServerImpl(dagClientHandler, FileSystem.get(conf)));
    when(dagClientHandler.getACLManager()).thenReturn(aclManager);
    when(dagClientHandler.submitDAG((DAGPlan) any(), (Map<String, LocalResource>) any())).thenReturn("dag-id");
    when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true);
    requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath());
    serverImpl.submitDAG(null, requestBuilder.build());
    ArgumentCaptor<DAGPlan> dagPlanCaptor = ArgumentCaptor.forClass(DAGPlan.class);
    verify(dagClientHandler).submitDAG(dagPlanCaptor.capture(), localResourcesCaptor.capture());
    dagPlan = dagPlanCaptor.getValue();
    localResources = localResourcesCaptor.getValue();
    assertEquals(dagPlan.getName(), dagPlanName);
    assertEquals(dagPlan.getVertexCount(), 1);
    assertTrue(dagPlan.getSerializedSize() > maxIPCMsgSize);
    assertArrayEquals(randomBytes, dagPlan.getVertex(0).getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
    assertEquals(localResources.size(), 1);
    assertTrue(localResources.containsKey(lrName));
    localResource = localResources.get(lrName);
    assertEquals(localResource.getType(), LocalResourceType.FILE);
    assertEquals(localResource.getVisibility(), LocalResourceVisibility.PUBLIC);
    lrURL = localResource.getResource();
    assertEquals(lrURL.getScheme(), scheme);
    assertEquals(lrURL.getHost(), host);
    assertEquals(lrURL.getPort(), port);
    assertEquals(lrURL.getFile(), path);
}

Example 3 with DAGClientHandler

use of org.apache.tez.dag.api.client.DAGClientHandler in project tez by apache.

the class LocalClient method createDAGAppMaster.

@SuppressWarnings("deprecation")
protected Thread createDAGAppMaster(final ApplicationSubmissionContext appContext) {
    Thread thread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                ApplicationId appId = appContext.getApplicationId();
                // Set up working directory for DAGAppMaster
                Path staging = TezCommonUtils.getTezSystemStagingPath(conf, appId.toString());
                Path userDir = TezCommonUtils.getTezSystemStagingPath(conf, appId.toString() + "_wd");
                LOG.info("Using working directory: " + userDir.toUri().getPath());
                FileSystem fs = FileSystem.get(conf);
                // copy data from staging directory to working directory to simulate the resource localizing
                FileUtil.copy(fs, staging, fs, userDir, false, conf);
                // Prepare Environment
                Path logDir = new Path(userDir, "localmode-log-dir");
                Path localDir = new Path(userDir, "localmode-local-dir");
                fs.mkdirs(logDir);
                fs.mkdirs(localDir);
                UserGroupInformation.setConfiguration(conf);
                // Add session specific credentials to the AM credentials.
                ByteBuffer tokens = appContext.getAMContainerSpec().getTokens();
                Credentials amCredentials;
                if (tokens != null) {
                    amCredentials = TezCommonUtils.parseCredentialsBytes(tokens.array());
                } else {
                    amCredentials = new Credentials();
                }
                // Construct, initialize, and start the DAGAppMaster
                ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 0);
                ContainerId cId = ContainerId.newInstance(applicationAttemptId, 1);
                String currentHost = InetAddress.getLocalHost().getHostName();
                int nmPort = YarnConfiguration.DEFAULT_NM_PORT;
                int nmHttpPort = YarnConfiguration.DEFAULT_NM_WEBAPP_PORT;
                long appSubmitTime = System.currentTimeMillis();
                dagAppMaster = createDAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, new SystemClock(), appSubmitTime, isSession, userDir.toUri().getPath(), new String[] { localDir.toUri().getPath() }, new String[] { logDir.toUri().getPath() }, amCredentials, UserGroupInformation.getCurrentUser().getShortUserName());
                DAGAppMaster.initAndStartAppMaster(dagAppMaster, conf);
                clientHandler = new DAGClientHandler(dagAppMaster);
            } catch (Throwable t) {
                LOG.error("Error starting DAGAppMaster", t);
                if (dagAppMaster != null) {
                    dagAppMaster.stop();
                    dagAppMaster = null;
                }
                amFailException = t;
            }
        }
    });
    thread.setName("DAGAppMaster Thread");
    LOG.info("DAGAppMaster thread has been created");
    return thread;
}