Examples with Application org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application used on opensource projects

Example 16 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class ContainerLogsUtils method getApplicationForContainer.

private static Application getApplicationForContainer(ContainerId containerId, Context context) {
    ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
    Application application = context.getApplications().get(applicationId);
    if (application == null) {
        throw new NotFoundException("Unknown container. Container either has not started or " + "has already completed or " + "doesn't belong to this node at all.");
    }
    return application;
}

Example 17 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class NMWebServices method getLogs.

/**
   * Returns the contents of a container's log file in plain text. 
   *
   * Only works for containers that are still in the NodeManager's memory, so
   * logs are no longer available after the corresponding application is no
   * longer running.
   * 
   * @param containerIdStr
   *    The container ID
   * @param filename
   *    The name of the log file
   * @param format
   *    The content type
   * @param size
   *    the size of the log file
   * @return
   *    The contents of the container's log file
   */
@GET
@Path("/containerlogs/{containerid}/{filename}")
@Produces({ MediaType.TEXT_PLAIN + "; " + JettyUtils.UTF_8 })
@Public
@Unstable
public Response getLogs(@PathParam(YarnWebServiceParams.CONTAINER_ID) final String containerIdStr, @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size) {
    ContainerId tempContainerId;
    try {
        tempContainerId = ContainerId.fromString(containerIdStr);
    } catch (IllegalArgumentException ex) {
        return Response.status(Status.BAD_REQUEST).build();
    }
    final ContainerId containerId = tempContainerId;
    boolean tempIsRunning = false;
    // check what is the status for container
    try {
        Container container = nmContext.getContainers().get(containerId);
        tempIsRunning = (container.getContainerState() == ContainerState.RUNNING);
    } catch (Exception ex) {
        // assume the container has already finished.
        if (LOG.isDebugEnabled()) {
            LOG.debug("Can not find the container:" + containerId + " in this node.");
        }
    }
    final boolean isRunning = tempIsRunning;
    File logFile = null;
    try {
        logFile = ContainerLogsUtils.getContainerLogFile(containerId, filename, request.getRemoteUser(), nmContext);
    } catch (NotFoundException ex) {
        if (redirectWSUrl == null || redirectWSUrl.isEmpty()) {
            return Response.status(Status.NOT_FOUND).entity(ex.getMessage()).build();
        }
        // redirect the request to the configured log server
        String redirectURI = "/containers/" + containerIdStr + "/logs/" + filename;
        return createRedirectResponse(request, redirectWSUrl, redirectURI);
    } catch (YarnException ex) {
        return Response.serverError().entity(ex.getMessage()).build();
    }
    final long bytes = parseLongParam(size);
    final String lastModifiedTime = Times.format(logFile.lastModified());
    final String outputFileName = filename;
    String contentType = WebAppUtils.getDefaultLogContentType();
    if (format != null && !format.isEmpty()) {
        contentType = WebAppUtils.getSupportedLogContentType(format);
        if (contentType == null) {
            String errorMessage = "The valid values for the parameter : format " + "are " + WebAppUtils.listSupportedLogContentType();
            return Response.status(Status.BAD_REQUEST).entity(errorMessage).build();
        }
    }
    try {
        final FileInputStream fis = ContainerLogsUtils.openLogFileForRead(containerIdStr, logFile, nmContext);
        final long fileLength = logFile.length();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream os) throws IOException, WebApplicationException {
                try {
                    int bufferSize = 65536;
                    byte[] buf = new byte[bufferSize];
                    LogToolUtils.outputContainerLog(containerId.toString(), nmContext.getNodeId().toString(), outputFileName, fileLength, bytes, lastModifiedTime, fis, os, buf, ContainerLogAggregationType.LOCAL);
                    StringBuilder sb = new StringBuilder();
                    String endOfFile = "End of LogType:" + outputFileName;
                    sb.append(endOfFile + ".");
                    if (isRunning) {
                        sb.append("This log file belongs to a running container (" + containerIdStr + ") and so may not be complete." + "\n");
                    } else {
                        sb.append("\n");
                    }
                    sb.append(StringUtils.repeat("*", endOfFile.length() + 50) + "\n\n");
                    os.write(sb.toString().getBytes(Charset.forName("UTF-8")));
                    // If we have aggregated logs for this container,
                    // output the aggregation logs as well.
                    ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
                    Application app = nmContext.getApplications().get(appId);
                    String appOwner = app == null ? null : app.getUser();
                    try {
                        LogToolUtils.outputAggregatedContainerLog(nmContext.getConf(), appId, appOwner, containerId.toString(), nmContext.getNodeId().toString(), outputFileName, bytes, os, buf);
                    } catch (Exception ex) {
                        // Something wrong when we try to access the aggregated log.
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Can not access the aggregated log for " + "the container:" + containerId);
                            LOG.debug(ex.getMessage());
                        }
                    }
                } finally {
                    IOUtils.closeQuietly(fis);
                }
            }
        };
        ResponseBuilder resp = Response.ok(stream);
        resp.header("Content-Type", contentType + "; " + JettyUtils.UTF_8);
        // Sending the X-Content-Type-Options response header with the value
        // nosniff will prevent Internet Explorer from MIME-sniffing a response
        // away from the declared content-type.
        resp.header("X-Content-Type-Options", "nosniff");
        return resp.build();
    } catch (IOException ex) {
        return Response.serverError().entity(ex.getMessage()).build();
    }
}

Example 18 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class NMWebServices method getNodeApp.

@GET
@Path("/apps/{appid}")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public AppInfo getNodeApp(@PathParam("appid") String appId) {
    init();
    ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId);
    Application app = this.nmContext.getApplications().get(id);
    if (app == null) {
        throw new NotFoundException("app with id " + appId + " not found");
    }
    return new AppInfo(app);
}

Example 19 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class NMWebServices method getContainerLogsInfo.

/**
   * Returns log file's name as well as current file size for a container.
   *
   * @param hsr
   *    HttpServletRequest
   * @param res
   *    HttpServletResponse
   * @param containerIdStr
   *    The container ID
   * @return
   *    The log file's name and current file size
   */
@GET
@Path("/containers/{containerid}/logs")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public Response getContainerLogsInfo(@javax.ws.rs.core.Context HttpServletRequest hsr, @javax.ws.rs.core.Context HttpServletResponse res, @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr) {
    ContainerId containerId = null;
    init();
    try {
        containerId = ContainerId.fromString(containerIdStr);
    } catch (IllegalArgumentException ex) {
        throw new BadRequestException("invalid container id, " + containerIdStr);
    }
    try {
        List<ContainerLogsInfo> containersLogsInfo = new ArrayList<>();
        containersLogsInfo.add(new NMContainerLogsInfo(this.nmContext, containerId, hsr.getRemoteUser(), ContainerLogAggregationType.LOCAL));
        // check whether we have aggregated logs in RemoteFS. If exists, show the
        // the log meta for the aggregated logs as well.
        ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
        Application app = this.nmContext.getApplications().get(appId);
        String appOwner = app == null ? null : app.getUser();
        try {
            List<ContainerLogMeta> containerLogMeta = LogToolUtils.getContainerLogMetaFromRemoteFS(this.nmContext.getConf(), appId, containerIdStr, this.nmContext.getNodeId().toString(), appOwner);
            if (!containerLogMeta.isEmpty()) {
                for (ContainerLogMeta logMeta : containerLogMeta) {
                    containersLogsInfo.add(new ContainerLogsInfo(logMeta, ContainerLogAggregationType.AGGREGATED));
                }
            }
        } catch (IOException ex) {
            // Skip it and do nothing
            if (LOG.isDebugEnabled()) {
                LOG.debug(ex.getMessage());
            }
        }
        GenericEntity<List<ContainerLogsInfo>> meta = new GenericEntity<List<ContainerLogsInfo>>(containersLogsInfo) {
        };
        ResponseBuilder resp = Response.ok(meta);
        // Sending the X-Content-Type-Options response header with the value
        // nosniff will prevent Internet Explorer from MIME-sniffing a response
        // away from the declared content-type.
        resp.header("X-Content-Type-Options", "nosniff");
        return resp.build();
    } catch (Exception ex) {
        if (redirectWSUrl == null || redirectWSUrl.isEmpty()) {
            throw new WebApplicationException(ex);
        }
        // redirect the request to the configured log server
        String redirectURI = "/containers/" + containerIdStr + "/logs";
        return createRedirectResponse(hsr, redirectWSUrl, redirectURI);
    }
}

Example 20 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class TestResourceLocalizationService method testPublicResourceInitializesLocalDir.

@Test
@SuppressWarnings("unchecked")
public void testPublicResourceInitializesLocalDir() throws Exception {
    // Setup state to simulate restart NM with existing state meaning no
    // directory creation during initialization
    NMStateStoreService spyStateStore = spy(nmContext.getNMStateStore());
    when(spyStateStore.canRecover()).thenReturn(true);
    NMContext spyContext = spy(nmContext);
    when(spyContext.getNMStateStore()).thenReturn(spyStateStore);
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    DrainDispatcher dispatcher = new DrainDispatcher();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = mock(DeletionService.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    dispatcher.init(conf);
    dispatcher.start();
    try {
        ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, spyContext);
        ResourceLocalizationService spyService = spy(rawService);
        doReturn(mockServer).when(spyService).createServer();
        doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
        spyService.init(conf);
        final FsPermission defaultPerm = new FsPermission((short) 0755);
        // verify directory is not created at initialization
        for (Path p : localDirs) {
            p = new Path((new URI(p.toString())).getPath());
            Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
            verify(spylfs, never()).mkdir(eq(publicCache), eq(defaultPerm), eq(true));
        }
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // Queue up public resource localization
        final LocalResource pubResource1 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq1 = new LocalResourceRequest(pubResource1);
        LocalResource pubResource2 = null;
        do {
            pubResource2 = getPublicMockedResource(r);
        } while (pubResource2 == null || pubResource2.equals(pubResource1));
        // above call to make sure we don't get identical resources.
        final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2);
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq1);
        pubRsrcs.add(pubReq2);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PUBLIC, pubRsrcs);
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        verify(spyService, times(1)).checkAndInitializeLocalDirs();
        // verify directory creation
        for (Path p : localDirs) {
            p = new Path((new URI(p.toString())).getPath());
            Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
            verify(spylfs).mkdir(eq(publicCache), eq(defaultPerm), eq(true));
        }
    } finally {
        dispatcher.stop();
    }
}