Examples with Application org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application used on opensource projects

Example 6 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class ContainerManagerImpl method startContainerInternal.

@SuppressWarnings("unchecked")
protected void startContainerInternal(ContainerTokenIdentifier containerTokenIdentifier, StartContainerRequest request) throws YarnException, IOException {
    ContainerId containerId = containerTokenIdentifier.getContainerID();
    String containerIdStr = containerId.toString();
    String user = containerTokenIdentifier.getApplicationSubmitter();
    LOG.info("Start request for " + containerIdStr + " by user " + user);
    ContainerLaunchContext launchContext = request.getContainerLaunchContext();
    Credentials credentials = YarnServerSecurityUtils.parseCredentials(launchContext);
    Container container = new ContainerImpl(getConfig(), this.dispatcher, launchContext, credentials, metrics, containerTokenIdentifier, context);
    ApplicationId applicationID = containerId.getApplicationAttemptId().getApplicationId();
    if (context.getContainers().putIfAbsent(containerId, container) != null) {
        NMAuditLogger.logFailure(user, AuditConstants.START_CONTAINER, "ContainerManagerImpl", "Container already running on this node!", applicationID, containerId);
        throw RPCUtil.getRemoteException("Container " + containerIdStr + " already is running on this node!!");
    }
    this.readLock.lock();
    try {
        if (!isServiceStopped()) {
            // Create the application
            // populate the flow context from the launch context if the timeline
            // service v.2 is enabled
            FlowContext flowContext = null;
            if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
                String flowName = launchContext.getEnvironment().get(TimelineUtils.FLOW_NAME_TAG_PREFIX);
                String flowVersion = launchContext.getEnvironment().get(TimelineUtils.FLOW_VERSION_TAG_PREFIX);
                String flowRunIdStr = launchContext.getEnvironment().get(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX);
                long flowRunId = 0L;
                if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) {
                    flowRunId = Long.parseLong(flowRunIdStr);
                }
                flowContext = new FlowContext(flowName, flowVersion, flowRunId);
            }
            if (!context.getApplications().containsKey(applicationID)) {
                Application application = new ApplicationImpl(dispatcher, user, flowContext, applicationID, credentials, context);
                if (context.getApplications().putIfAbsent(applicationID, application) == null) {
                    LOG.info("Creating a new application reference for app " + applicationID);
                    LogAggregationContext logAggregationContext = containerTokenIdentifier.getLogAggregationContext();
                    Map<ApplicationAccessType, String> appAcls = container.getLaunchContext().getApplicationACLs();
                    context.getNMStateStore().storeApplication(applicationID, buildAppProto(applicationID, user, credentials, appAcls, logAggregationContext));
                    dispatcher.getEventHandler().handle(new ApplicationInitEvent(applicationID, appAcls, logAggregationContext));
                }
            }
            this.context.getNMStateStore().storeContainer(containerId, containerTokenIdentifier.getVersion(), request);
            dispatcher.getEventHandler().handle(new ApplicationContainerInitEvent(container));
            this.context.getContainerTokenSecretManager().startContainerSuccessful(containerTokenIdentifier);
            NMAuditLogger.logSuccess(user, AuditConstants.START_CONTAINER, "ContainerManageImpl", applicationID, containerId);
            // TODO launchedContainer misplaced -> doesn't necessarily mean a container
            // launch. A finished Application will not launch containers.
            metrics.launchedContainer();
            metrics.allocateContainer(containerTokenIdentifier.getResource());
        } else {
            throw new YarnException("Container start failed as the NodeManager is " + "in the process of shutting down");
        }
    } finally {
        this.readLock.unlock();
    }
}

Example 7 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class ContainerManagerImpl method handle.

@SuppressWarnings("unchecked")
@Override
public void handle(ContainerManagerEvent event) {
    switch(event.getType()) {
        case FINISH_APPS:
            CMgrCompletedAppsEvent appsFinishedEvent = (CMgrCompletedAppsEvent) event;
            for (ApplicationId appID : appsFinishedEvent.getAppsToCleanup()) {
                Application app = this.context.getApplications().get(appID);
                if (app == null) {
                    LOG.warn("couldn't find application " + appID + " while processing" + " FINISH_APPS event");
                    continue;
                }
                boolean shouldDropEvent = false;
                for (Container container : app.getContainers().values()) {
                    if (container.isRecovering()) {
                        LOG.info("drop FINISH_APPS event to " + appID + " because " + "container " + container.getContainerId() + " is recovering");
                        shouldDropEvent = true;
                        break;
                    }
                }
                if (shouldDropEvent) {
                    continue;
                }
                String diagnostic = "";
                if (appsFinishedEvent.getReason() == CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN) {
                    diagnostic = "Application killed on shutdown";
                } else if (appsFinishedEvent.getReason() == CMgrCompletedAppsEvent.Reason.BY_RESOURCEMANAGER) {
                    diagnostic = "Application killed by ResourceManager";
                }
                this.dispatcher.getEventHandler().handle(new ApplicationFinishEvent(appID, diagnostic));
            }
            break;
        case FINISH_CONTAINERS:
            CMgrCompletedContainersEvent containersFinishedEvent = (CMgrCompletedContainersEvent) event;
            for (ContainerId containerId : containersFinishedEvent.getContainersToCleanup()) {
                ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
                Application app = this.context.getApplications().get(appId);
                if (app == null) {
                    LOG.warn("couldn't find app " + appId + " while processing" + " FINISH_CONTAINERS event");
                    continue;
                }
                Container container = app.getContainers().get(containerId);
                if (container == null) {
                    LOG.warn("couldn't find container " + containerId + " while processing FINISH_CONTAINERS event");
                    continue;
                }
                if (container.isRecovering()) {
                    LOG.info("drop FINISH_CONTAINERS event to " + containerId + " because container is recovering");
                    continue;
                }
                this.dispatcher.getEventHandler().handle(new ContainerKillEvent(containerId, ContainerExitStatus.KILLED_BY_RESOURCEMANAGER, "Container Killed by ResourceManager"));
            }
            break;
        case DECREASE_CONTAINERS_RESOURCE:
            CMgrDecreaseContainersResourceEvent containersDecreasedEvent = (CMgrDecreaseContainersResourceEvent) event;
            for (org.apache.hadoop.yarn.api.records.Container container : containersDecreasedEvent.getContainersToDecrease()) {
                try {
                    changeContainerResourceInternal(container.getId(), container.getVersion(), container.getResource(), false);
                } catch (YarnException e) {
                    LOG.error("Unable to decrease container resource", e);
                } catch (IOException e) {
                    LOG.error("Unable to update container resource in store", e);
                }
            }
            break;
        case SIGNAL_CONTAINERS:
            CMgrSignalContainersEvent containersSignalEvent = (CMgrSignalContainersEvent) event;
            for (SignalContainerRequest request : containersSignalEvent.getContainersToSignal()) {
                internalSignalToContainer(request, "ResourceManager");
            }
            break;
        default:
            throw new YarnRuntimeException("Got an unknown ContainerManagerEvent type: " + event.getType());
    }
}

Example 8 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class TestNMWebServicesContainers method testNodeSingleContainerXML.

@Test
public void testNodeSingleContainerXML() throws JSONException, Exception {
    WebResource r = resource();
    Application app = new MockApp(1);
    nmContext.getApplications().put(app.getAppId(), app);
    HashMap<String, String> hash = addAppContainers(app);
    Application app2 = new MockApp(2);
    nmContext.getApplications().put(app2.getAppId(), app2);
    addAppContainers(app2);
    client().addFilter(new LoggingFilter(System.out));
    for (String id : hash.keySet()) {
        ClientResponse response = r.path("ws").path("v1").path("node").path("containers").path(id).accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        String xml = response.getEntity(String.class);
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        DocumentBuilder db = dbf.newDocumentBuilder();
        InputSource is = new InputSource();
        is.setCharacterStream(new StringReader(xml));
        Document dom = db.parse(is);
        NodeList nodes = dom.getElementsByTagName("container");
        assertEquals("incorrect number of elements", 1, nodes.getLength());
        verifyContainersInfoXML(nodes, nmContext.getContainers().get(ContainerId.fromString(id)));
    }
}

Example 9 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class TestNMWebServicesContainers method testSingleContainerWrong.

@Test
public void testSingleContainerWrong() throws JSONException, Exception {
    WebResource r = resource();
    Application app = new MockApp(1);
    nmContext.getApplications().put(app.getAppId(), app);
    addAppContainers(app);
    Application app2 = new MockApp(2);
    nmContext.getApplications().put(app2.getAppId(), app2);
    addAppContainers(app2);
    try {
        r.path("ws").path("v1").path("node").path("containers").path("container_1234_0001_01_000005").accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
        fail("should have thrown exception on invalid user query");
    } catch (UniformInterfaceException ue) {
        ClientResponse response = ue.getResponse();
        assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo());
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        JSONObject msg = response.getEntity(JSONObject.class);
        JSONObject exception = msg.getJSONObject("RemoteException");
        assertEquals("incorrect number of elements", 3, exception.length());
        String message = exception.getString("message");
        String type = exception.getString("exception");
        String classname = exception.getString("javaClassName");
        WebServicesTestUtils.checkStringMatch("exception message", "java.lang.Exception: container with id, container_1234_0001_01_000005, not found", message);
        WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type);
        WebServicesTestUtils.checkStringMatch("exception classname", "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
    }
}

Example 10 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class TestNMWebServicesApps method testNodeAppsStateNone.

@Test
public void testNodeAppsStateNone() throws JSONException, Exception {
    WebResource r = resource();
    Application app = new MockApp(1);
    nmContext.getApplications().put(app.getAppId(), app);
    addAppContainers(app);
    Application app2 = new MockApp("foo", 1234, 2);
    nmContext.getApplications().put(app2.getAppId(), app2);
    addAppContainers(app2);
    ClientResponse response = r.path("ws").path("v1").path("node").path("apps").queryParam("state", ApplicationState.INITING.toString()).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("apps is not empty", new JSONObject().toString(), json.get("apps").toString());
}