Examples with ContainerRequest org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest used on opensource projects

Example 36 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project apex-core by apache.

the class StreamingAppMasterService method sendContainerAskToRM.

/**
 * Ask RM to allocate given no. of containers to this Application Master
 *
 * @param containerRequests        Containers to ask for from RM
 * @param removedContainerRequests Container requests to be removed
 * @param releasedContainers
 * @return Response from RM to AM with allocated containers
 * @throws YarnException
 */
private AllocateResponse sendContainerAskToRM(List<ContainerRequest> containerRequests, List<ContainerRequest> removedContainerRequests, List<ContainerId> releasedContainers) throws YarnException, IOException {
    if (removedContainerRequests.size() > 0) {
        LOG.debug("Removing container request: {}", removedContainerRequests);
        for (ContainerRequest cr : removedContainerRequests) {
            amRmClient.removeContainerRequest(cr);
        }
    }
    if (containerRequests.size() > 0) {
        LOG.debug("Asking RM for containers: {}", containerRequests);
        for (ContainerRequest cr : containerRequests) {
            amRmClient.addContainerRequest(cr);
        }
    }
    for (ContainerId containerId : releasedContainers) {
        LOG.info("Released container, id={}", containerId.getId());
        amRmClient.releaseAssignedContainer(containerId);
    }
    for (String containerIdStr : dnmgr.containerStopRequests.values()) {
        AllocatedContainer allocatedContainer = this.allocatedContainers.get(containerIdStr);
        if (allocatedContainer != null) {
            allocatedContainer.stop(nmClient);
        }
        dnmgr.containerStopRequests.remove(containerIdStr);
    }
    for (Map.Entry<String, AllocatedContainer> entry : allocatedContainers.entrySet()) {
        if (entry.getValue().checkStopRequestedTimeout()) {
            LOG.info("Timeout happened for NodeManager kill container request, recovering the container {} without waiting", entry.getKey());
            recoverHelper(entry.getKey());
        }
    }
    return amRmClient.allocate(0);
}

Example 37 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project apex-core by apache.

the class ResourceRequestHandler method reissueContainerRequests.

/**
 * Issue requests to AM RM Client again if previous container requests expired and were not allocated by Yarn
 * @param amRmClient
 * @param requestedResources
 * @param loopCounter
 * @param resourceRequestor
 * @param containerRequests
 * @param removedContainerRequests
 */
public void reissueContainerRequests(AMRMClient<ContainerRequest> amRmClient, Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> containerRequests, List<ContainerRequest> removedContainerRequests) {
    if (!requestedResources.isEmpty()) {
        for (Map.Entry<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> entry : requestedResources.entrySet()) {
            /*
         * Create container requests again if pending requests were not allocated by Yarn till timeout.
         */
            if ((loopCounter - entry.getValue().getKey()) > NUMBER_MISSED_HEARTBEATS) {
                StreamingContainerAgent.ContainerStartRequest csr = entry.getKey();
                LOG.debug("Request for container {} timed out. Re-requesting container", csr.container);
                removedContainerRequests.add(entry.getValue().getRight());
                ContainerRequest cr = resourceRequestor.createContainerRequest(csr, false);
                entry.getValue().setLeft(loopCounter);
                entry.getValue().setRight(cr);
                containerRequests.add(cr);
            }
        }
    }
}

Example 38 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project jstorm by alibaba.

the class JstormMaster method setupContainerAskForRM.

/**
 * Setup the request that will be sent to the RM for the container ask.
 *
 * @return the setup ResourceRequest to be sent to RM
 */
public ContainerRequest setupContainerAskForRM(int containerMemory, int containerVirtualCores, int priority, String host) {
    // setup requirements for hosts
    // using * as any host will do for the jstorm app
    // set the priority for the request
    Priority pri = Priority.newInstance(priority);
    // Set up resource type requirements
    // For now, memory and CPU are supported so we set memory and cpu requirements
    Resource capability = Resource.newInstance(containerMemory, containerVirtualCores);
    ContainerRequest request = new ContainerRequest(capability, null, null, pri);
    LOG.info("By Thrift Server Requested container  ask: " + request.toString());
    return request;
}

Example 39 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project jstorm by alibaba.

the class JstormMaster method run.

/**
 * Main run function for the application master
 *
 * @throws YarnException
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public void run() throws Exception {
    LOG.info("Starting JstormMaster");
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token);
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    jstormMasterContext.allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    // Create appSubmitterUgi and add original tokens to it
    String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name());
    appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName);
    appSubmitterUgi.addCredentials(credentials);
    AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
    amRMClient = AMRMClientAsync.createAMRMClientAsync(JOYConstants.AM_RM_CLIENT_INTERVAL, allocListener);
    jstormMasterContext.amRMClient = amRMClient;
    amRMClient.init(conf);
    amRMClient.start();
    containerListener = createNMCallbackHandler();
    nmClientAsync = new NMClientAsyncImpl(containerListener);
    nmClientAsync.init(conf);
    nmClientAsync.start();
    startTimelineClient(conf);
    if (timelineClient != null) {
        publishApplicationAttemptEvent(timelineClient, jstormMasterContext.appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, jstormMasterContext.domainId, appSubmitterUgi);
    }
    // Register self with ResourceManager
    // This will start heartbeating to the RM
    jstormMasterContext.appMasterHostname = NetUtils.getHostname();
    // get available port
    buildPortScanner();
    jstormMasterContext.appMasterThriftPort = portScanner.getAvailablePort();
    // since appMasterRpcPort not used yet,  set appMasterRpcPort to appMasterThriftPort
    jstormMasterContext.appMasterRpcPort = jstormMasterContext.appMasterThriftPort;
    RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(jstormMasterContext.appMasterHostname, jstormMasterContext.appMasterRpcPort, jstormMasterContext.appMasterTrackingUrl);
    // Dump out information about cluster capability as seen by the
    // resource manager
    jstormMasterContext.maxMemory = response.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capability of resources in this cluster " + jstormMasterContext.maxMemory);
    jstormMasterContext.maxVcores = response.getMaximumResourceCapability().getVirtualCores();
    LOG.info("Max vcores capability of resources in this cluster " + jstormMasterContext.maxVcores);
    // A resource ask cannot exceed the max.
    if (jstormMasterContext.containerMemory > jstormMasterContext.maxMemory) {
        LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + jstormMasterContext.containerMemory + ", max=" + jstormMasterContext.maxMemory);
        jstormMasterContext.containerMemory = jstormMasterContext.maxMemory;
    }
    if (jstormMasterContext.containerVirtualCores > jstormMasterContext.maxVcores) {
        LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + jstormMasterContext.containerVirtualCores + ", max=" + jstormMasterContext.maxVcores);
        jstormMasterContext.containerVirtualCores = jstormMasterContext.maxVcores;
    }
    List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts();
    LOG.info(jstormMasterContext.appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration.");
    jstormMasterContext.numAllocatedContainers.addAndGet(previousAMRunningContainers.size());
    // Setup RegistryOperations
    registryOperations = RegistryOperationsFactory.createInstance(JOYConstants.YARN_REGISTRY, conf);
    setupInitialRegistryPaths();
    registryOperations.start();
    // add previous AM containers to supervisor and nimbus container list
    for (Container container : previousAMRunningContainers) {
        String containerPath = RegistryUtils.componentPath(JOYConstants.APP_TYPE, jstormMasterContext.instanceName, container.getId().getApplicationAttemptId().getApplicationId().toString(), container.getId().toString());
        ServiceRecord sr = null;
        try {
            if (!registryOperations.exists(containerPath)) {
                String contianerHost = container.getNodeId().getHost();
                registryOperations.mknode(containerPath, true);
                sr = new ServiceRecord();
                sr.set(JOYConstants.HOST, contianerHost);
                sr.set(YarnRegistryAttributes.YARN_ID, container.getId().toString());
                sr.description = JOYConstants.CONTAINER;
                sr.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.CONTAINER);
                registryOperations.bind(containerPath, sr, BindFlags.OVERWRITE);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        if (container.getPriority().getPriority() == 0)
            jstormMasterContext.supervisorContainers.add(container);
        else if (container.getPriority().getPriority() == 1) {
            jstormMasterContext.nimbusContainers.add(container);
        }
    }
    jstormMasterContext.requestBlockingQueue = new LinkedBlockingQueue<ContainerRequest>();
    jstormMasterContext.service_user_name = RegistryUtils.currentUser();
    jstormMasterContext.instanceName = conf.get(JOYConstants.INSTANCE_NAME_KEY);
    this.jstormMasterContext.user = conf.get(JOYConstants.JSTORM_YARN_USER);
    this.jstormMasterContext.password = conf.get(JOYConstants.JSTORM_YARN_PASSWORD);
    this.jstormMasterContext.oldPassword = conf.get(JOYConstants.JSTORM_YARN_OLD_PASSWORD);
    LOG.info("find available port for am rpc server which is : " + jstormMasterContext.appMasterThriftPort);
    String appPath = RegistryUtils.servicePath(JOYConstants.APP_TYPE, jstormMasterContext.instanceName, jstormMasterContext.appAttemptID.getApplicationId().toString());
    String instancePath = RegistryUtils.serviceclassPath(JOYConstants.APP_TYPE, jstormMasterContext.instanceName);
    LOG.info("Registering application " + jstormMasterContext.appAttemptID.getApplicationId().toString());
    ServiceRecord application = setupServiceRecord();
    jstormMasterContext.nimbusDataDirPrefix = conf.get(JOYConstants.INSTANCE_DATA_DIR_KEY);
    LOG.info("generate instancePath on zk , path is:" + instancePath);
    if (registryOperations.exists(instancePath)) {
        ServiceRecord previousRegister = registryOperations.resolve(instancePath);
        application.set(JOYConstants.NIMBUS_HOST, previousRegister.get(JOYConstants.NIMBUS_HOST, JOYConstants.EMPTY));
        application.set(JOYConstants.NIMBUS_CONTAINER, previousRegister.get(JOYConstants.NIMBUS_CONTAINER, JOYConstants.EMPTY));
        application.set(JOYConstants.NIMBUS_LOCAL_DIR, previousRegister.get(JOYConstants.NIMBUS_LOCAL_DIR, JOYConstants.EMPTY));
        jstormMasterContext.previousNimbusHost = previousRegister.get(JOYConstants.NIMBUS_HOST, "");
        Date now = new Date();
        Map<String, ServiceRecord> apps = RegistryUtils.listServiceRecords(registryOperations, instancePath);
        for (String subAppPath : apps.keySet()) {
            LOG.info("existApp:" + subAppPath);
            ServiceRecord subApp = apps.get(subAppPath);
            Long lastHeatBeatTime = 0l;
            try {
                lastHeatBeatTime = Long.parseLong(subApp.get(JOYConstants.APP_HEARTBEAT_TIME));
            } catch (Exception e) {
                LOG.error(e);
            }
            if (now.getTime() - lastHeatBeatTime > 5 * JOYConstants.HEARTBEAT_TIME_INTERVAL || lastHeatBeatTime > now.getTime() || subAppPath.trim().equals(appPath.trim())) {
                LOG.info("application " + subAppPath + " not response , delete it!");
                registryOperations.delete(subAppPath, true);
            }
        }
    }
    if (!jstormMasterContext.done) {
        jstormMasterContext.config = conf;
        registryOperations.mknode(appPath, true);
        registryOperations.bind(instancePath, application, BindFlags.OVERWRITE);
        ServiceRecord previousRegister = registryOperations.resolve(instancePath);
        LOG.info("previousRegister:" + previousRegister.toString());
        LOG.info("register path: " + instancePath);
        AMServer as = new AMServer(jstormMasterContext.appMasterThriftPort);
        as.Start(this);
    }
}

Example 40 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project jstorm by alibaba.

the class JstormMaster method setupContainerAskForRM.

/**
 * Setup the request that will be sent to the RM for the container ask.
 *
 * @return the setup ResourceRequest to be sent to RM
 */
public ContainerRequest setupContainerAskForRM(int containerMemory, int containerVirtualCores, int priority, String[] racks, String[] hosts) {
    Priority pri = Priority.newInstance(priority);
    Resource capability = Resource.newInstance(containerMemory, containerVirtualCores);
    ContainerRequest request = new ContainerRequest(capability, hosts, racks, pri, false);
    LOG.info("By Thrift Server Requested container  ask: " + request.toString());
    return request;
}