Examples with NMClientAsyncImpl org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl used on opensource projects

Example 1 with NMClientAsyncImpl

use of org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl in project jstorm by alibaba.

the class JstormMaster method run.

/**
     * Main run function for the application master
     *
     * @throws YarnException
     * @throws IOException
     */
@SuppressWarnings({ "unchecked" })
public void run() throws Exception {
    LOG.info("Starting JstormMaster");
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token);
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    jstormMasterContext.allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    // Create appSubmitterUgi and add original tokens to it
    String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name());
    appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName);
    appSubmitterUgi.addCredentials(credentials);
    AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
    amRMClient = AMRMClientAsync.createAMRMClientAsync(JOYConstants.AM_RM_CLIENT_INTERVAL, allocListener);
    jstormMasterContext.amRMClient = amRMClient;
    amRMClient.init(conf);
    amRMClient.start();
    containerListener = createNMCallbackHandler();
    nmClientAsync = new NMClientAsyncImpl(containerListener);
    nmClientAsync.init(conf);
    nmClientAsync.start();
    startTimelineClient(conf);
    if (timelineClient != null) {
        publishApplicationAttemptEvent(timelineClient, jstormMasterContext.appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, jstormMasterContext.domainId, appSubmitterUgi);
    }
    // Register self with ResourceManager
    // This will start heartbeating to the RM
    jstormMasterContext.appMasterHostname = NetUtils.getHostname();
    //get available port
    buildPortScanner();
    jstormMasterContext.appMasterThriftPort = portScanner.getAvailablePort();
    //since appMasterRpcPort not used yet,  set appMasterRpcPort to appMasterThriftPort
    jstormMasterContext.appMasterRpcPort = jstormMasterContext.appMasterThriftPort;
    RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(jstormMasterContext.appMasterHostname, jstormMasterContext.appMasterRpcPort, jstormMasterContext.appMasterTrackingUrl);
    // Dump out information about cluster capability as seen by the
    // resource manager
    jstormMasterContext.maxMemory = response.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capability of resources in this cluster " + jstormMasterContext.maxMemory);
    jstormMasterContext.maxVcores = response.getMaximumResourceCapability().getVirtualCores();
    LOG.info("Max vcores capability of resources in this cluster " + jstormMasterContext.maxVcores);
    // A resource ask cannot exceed the max.
    if (jstormMasterContext.containerMemory > jstormMasterContext.maxMemory) {
        LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + jstormMasterContext.containerMemory + ", max=" + jstormMasterContext.maxMemory);
        jstormMasterContext.containerMemory = jstormMasterContext.maxMemory;
    }
    if (jstormMasterContext.containerVirtualCores > jstormMasterContext.maxVcores) {
        LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + jstormMasterContext.containerVirtualCores + ", max=" + jstormMasterContext.maxVcores);
        jstormMasterContext.containerVirtualCores = jstormMasterContext.maxVcores;
    }
    List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts();
    LOG.info(jstormMasterContext.appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration.");
    jstormMasterContext.numAllocatedContainers.addAndGet(previousAMRunningContainers.size());
    //Setup RegistryOperations
    registryOperations = RegistryOperationsFactory.createInstance(JOYConstants.YARN_REGISTRY, conf);
    setupInitialRegistryPaths();
    registryOperations.start();
    //add previous AM containers to supervisor and nimbus container list
    for (Container container : previousAMRunningContainers) {
        String containerPath = RegistryUtils.componentPath(JOYConstants.APP_TYPE, jstormMasterContext.instanceName, container.getId().getApplicationAttemptId().getApplicationId().toString(), container.getId().toString());
        ServiceRecord sr = null;
        try {
            if (!registryOperations.exists(containerPath)) {
                String contianerHost = container.getNodeId().getHost();
                registryOperations.mknode(containerPath, true);
                sr = new ServiceRecord();
                sr.set(JOYConstants.HOST, contianerHost);
                sr.set(YarnRegistryAttributes.YARN_ID, container.getId().toString());
                sr.description = JOYConstants.CONTAINER;
                sr.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.CONTAINER);
                registryOperations.bind(containerPath, sr, BindFlags.OVERWRITE);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        if (container.getPriority().getPriority() == 0)
            jstormMasterContext.supervisorContainers.add(container);
        else if (container.getPriority().getPriority() == 1) {
            jstormMasterContext.nimbusContainers.add(container);
        }
    }
    jstormMasterContext.requestBlockingQueue = new LinkedBlockingQueue<ContainerRequest>();
    jstormMasterContext.service_user_name = RegistryUtils.currentUser();
    jstormMasterContext.instanceName = conf.get(JOYConstants.INSTANCE_NAME_KEY);
    this.jstormMasterContext.user = conf.get(JOYConstants.JSTORM_YARN_USER);
    this.jstormMasterContext.password = conf.get(JOYConstants.JSTORM_YARN_PASSWORD);
    this.jstormMasterContext.oldPassword = conf.get(JOYConstants.JSTORM_YARN_OLD_PASSWORD);
    LOG.info("find available port for am rpc server which is : " + jstormMasterContext.appMasterThriftPort);
    String appPath = RegistryUtils.servicePath(JOYConstants.APP_TYPE, jstormMasterContext.instanceName, jstormMasterContext.appAttemptID.getApplicationId().toString());
    String instancePath = RegistryUtils.serviceclassPath(JOYConstants.APP_TYPE, jstormMasterContext.instanceName);
    LOG.info("Registering application " + jstormMasterContext.appAttemptID.getApplicationId().toString());
    ServiceRecord application = setupServiceRecord();
    jstormMasterContext.nimbusDataDirPrefix = conf.get(JOYConstants.INSTANCE_DATA_DIR_KEY);
    LOG.info("generate instancePath on zk , path is:" + instancePath);
    if (registryOperations.exists(instancePath)) {
        ServiceRecord previousRegister = registryOperations.resolve(instancePath);
        application.set(JOYConstants.NIMBUS_HOST, previousRegister.get(JOYConstants.NIMBUS_HOST, JOYConstants.EMPTY));
        application.set(JOYConstants.NIMBUS_CONTAINER, previousRegister.get(JOYConstants.NIMBUS_CONTAINER, JOYConstants.EMPTY));
        application.set(JOYConstants.NIMBUS_LOCAL_DIR, previousRegister.get(JOYConstants.NIMBUS_LOCAL_DIR, JOYConstants.EMPTY));
        jstormMasterContext.previousNimbusHost = previousRegister.get(JOYConstants.NIMBUS_HOST, "");
        Date now = new Date();
        Map<String, ServiceRecord> apps = RegistryUtils.listServiceRecords(registryOperations, instancePath);
        for (String subAppPath : apps.keySet()) {
            LOG.info("existApp:" + subAppPath);
            ServiceRecord subApp = apps.get(subAppPath);
            Long lastHeatBeatTime = 0l;
            try {
                lastHeatBeatTime = Long.parseLong(subApp.get(JOYConstants.APP_HEARTBEAT_TIME));
            } catch (Exception e) {
                LOG.error(e);
            }
            if (now.getTime() - lastHeatBeatTime > 5 * JOYConstants.HEARTBEAT_TIME_INTERVAL || lastHeatBeatTime > now.getTime() || subAppPath.trim().equals(appPath.trim())) {
                LOG.info("application " + subAppPath + " not response , delete it!");
                registryOperations.delete(subAppPath, true);
            }
        }
    }
    if (!jstormMasterContext.done) {
        jstormMasterContext.config = conf;
        registryOperations.mknode(appPath, true);
        registryOperations.bind(instancePath, application, BindFlags.OVERWRITE);
        ServiceRecord previousRegister = registryOperations.resolve(instancePath);
        LOG.info("previousRegister:" + previousRegister.toString());
        LOG.info("register path: " + instancePath);
        AMServer as = new AMServer(jstormMasterContext.appMasterThriftPort);
        as.Start(this);
    }
}

Example 2 with NMClientAsyncImpl

use of org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl in project asterixdb by apache.

the class AsterixApplicationMaster method run.

/**
     * Start the AM and request all necessary resources.
     *
     * @return True if the run fully succeeded, false otherwise.
     * @throws YarnException
     * @throws IOException
     */
public boolean run() throws YarnException, IOException {
    LOG.info("Starting ApplicationMaster");
    AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
    resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
    resourceManager.init(conf);
    resourceManager.start();
    containerListener = new NMCallbackHandler();
    nmClientAsync = new NMClientAsyncImpl(containerListener);
    nmClientAsync.init(conf);
    nmClientAsync.start();
    // This will start heartbeating to the RM
    try {
        appMasterHostname = InetAddress.getLocalHost().toString();
    } catch (java.net.UnknownHostException uhe) {
        appMasterHostname = uhe.toString();
    }
    RegisterApplicationMasterResponse response = resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl);
    // Dump out information about cluster capability as seen by the
    // resource manager
    int maxMem = response.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
    try {
        requestResources(clusterDesc);
    } catch (YarnException e) {
        LOG.error("Could not allocate resources properly:" + e.getMessage());
        done = true;
        throw e;
    }
    while (!done) {
        try {
            Thread.sleep(200);
        } catch (InterruptedException ex) {
        }
    }
    finish();
    return success;
}

Example 3 with NMClientAsyncImpl

use of org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl in project hadoop by apache.

the class ApplicationMaster method run.

/**
   * Main run function for the application master
   *
   * @throws YarnException
   * @throws IOException
   */
@SuppressWarnings({ "unchecked" })
public void run() throws YarnException, IOException, InterruptedException {
    LOG.info("Starting ApplicationMaster");
    // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class
    // are marked as LimitedPrivate
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token);
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    // Create appSubmitterUgi and add original tokens to it
    String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name());
    appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName);
    appSubmitterUgi.addCredentials(credentials);
    AMRMClientAsync.AbstractCallbackHandler allocListener = new RMCallbackHandler();
    amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
    amRMClient.init(conf);
    amRMClient.start();
    containerListener = createNMCallbackHandler();
    nmClientAsync = new NMClientAsyncImpl(containerListener);
    nmClientAsync.init(conf);
    nmClientAsync.start();
    startTimelineClient(conf);
    if (timelineServiceV2Enabled) {
        // need to bind timelineClient
        amRMClient.registerTimelineV2Client(timelineV2Client);
    }
    if (timelineServiceV2Enabled) {
        publishApplicationAttemptEventOnTimelineServiceV2(DSEvent.DS_APP_ATTEMPT_START);
    } else if (timelineServiceV1Enabled) {
        publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
    }
    // Setup local RPC Server to accept status requests directly from clients
    // TODO need to setup a protocol for client to be able to communicate to
    // the RPC server
    // TODO use the rpc port info to register with the RM for the client to
    // send requests to this app master
    // Register self with ResourceManager
    // This will start heartbeating to the RM
    appMasterHostname = NetUtils.getHostname();
    RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl);
    // Dump out information about cluster capability as seen by the
    // resource manager
    long maxMem = response.getMaximumResourceCapability().getMemorySize();
    LOG.info("Max mem capability of resources in this cluster " + maxMem);
    int maxVCores = response.getMaximumResourceCapability().getVirtualCores();
    LOG.info("Max vcores capability of resources in this cluster " + maxVCores);
    // A resource ask cannot exceed the max.
    if (containerMemory > maxMem) {
        LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem);
        containerMemory = maxMem;
    }
    if (containerVirtualCores > maxVCores) {
        LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + containerVirtualCores + ", max=" + maxVCores);
        containerVirtualCores = maxVCores;
    }
    List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts();
    LOG.info(appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration.");
    for (Container container : previousAMRunningContainers) {
        launchedContainers.add(container.getId());
    }
    numAllocatedContainers.addAndGet(previousAMRunningContainers.size());
    int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size();
    // executed on them ( regardless of success/failure).
    for (int i = 0; i < numTotalContainersToRequest; ++i) {
        ContainerRequest containerAsk = setupContainerAskForRM();
        amRMClient.addContainerRequest(containerAsk);
    }
    numRequestedContainers.set(numTotalContainers);
}

Example 4 with NMClientAsyncImpl

use of org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl in project apex-core by apache.

the class StreamingAppMasterService method serviceInit.

@Override
protected void serviceInit(Configuration conf) throws Exception {
    LOG.info("Application master" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId());
    FileInputStream fis = new FileInputStream("./" + LogicalPlan.SER_FILE_NAME);
    try {
        this.dag = LogicalPlan.read(fis);
    } finally {
        fis.close();
    }
    // "debug" simply dumps all data using LOG.info
    if (dag.isDebug()) {
        dumpOutDebugInfo();
    }
    dag.setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, appAttemptID.getAttemptId());
    FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), conf);
    this.dnmgr = StreamingContainerManager.getInstance(recoveryHandler, dag, true);
    dag = this.dnmgr.getLogicalPlan();
    this.appContext = new ClusterAppContextImpl(dag.getAttributes());
    Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dag.getAttributes().get(DAG.STRING_CODECS);
    StringCodecs.loadConverters(codecs);
    LOG.info("Starting application with {} operators in {} containers", dnmgr.getPhysicalPlan().getAllOperators().size(), dnmgr.getPhysicalPlan().getContainers().size());
    // Setup security configuration such as that for web security
    SecurityUtils.init(conf, dag.getValue(LogicalPlan.STRAM_HTTP_AUTHENTICATION));
    if (UserGroupInformation.isSecurityEnabled()) {
        // TODO :- Need to perform token renewal
        delegationTokenManager = new StramDelegationTokenManager(DELEGATION_KEY_UPDATE_INTERVAL, DELEGATION_TOKEN_MAX_LIFETIME, DELEGATION_TOKEN_RENEW_INTERVAL, DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL);
    }
    this.nmClient = new NMClientAsyncImpl(new NMCallbackHandler());
    addService(nmClient);
    this.amRmClient = AMRMClient.createAMRMClient();
    addService(amRmClient);
    // start RPC server
    int rpcListenerCount = dag.getValue(DAGContext.HEARTBEAT_LISTENER_THREAD_COUNT);
    this.heartbeatListener = new StreamingContainerParent(this.getClass().getName(), dnmgr, delegationTokenManager, rpcListenerCount);
    addService(heartbeatListener);
    AutoMetric.Transport appDataPushTransport = dag.getValue(LogicalPlan.METRICS_TRANSPORT);
    if (appDataPushTransport != null) {
        this.appDataPushAgent = new AppDataPushAgent(dnmgr, appContext);
        addService(this.appDataPushAgent);
    }
    initApexPluginDispatcher();
    // Initialize all services added above
    super.serviceInit(conf);
}