Examples with Unstable org.apache.hadoop.classification.InterfaceStability.Unstable used on opensource projects

Example 1 with Unstable

use of org.apache.hadoop.classification.InterfaceStability.Unstable in project hadoop by apache.

the class NMWebServices method getLogs.

/**
   * Returns the contents of a container's log file in plain text. 
   *
   * Only works for containers that are still in the NodeManager's memory, so
   * logs are no longer available after the corresponding application is no
   * longer running.
   * 
   * @param containerIdStr
   *    The container ID
   * @param filename
   *    The name of the log file
   * @param format
   *    The content type
   * @param size
   *    the size of the log file
   * @return
   *    The contents of the container's log file
   */
@GET
@Path("/containerlogs/{containerid}/{filename}")
@Produces({ MediaType.TEXT_PLAIN + "; " + JettyUtils.UTF_8 })
@Public
@Unstable
public Response getLogs(@PathParam(YarnWebServiceParams.CONTAINER_ID) final String containerIdStr, @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size) {
    ContainerId tempContainerId;
    try {
        tempContainerId = ContainerId.fromString(containerIdStr);
    } catch (IllegalArgumentException ex) {
        return Response.status(Status.BAD_REQUEST).build();
    }
    final ContainerId containerId = tempContainerId;
    boolean tempIsRunning = false;
    // check what is the status for container
    try {
        Container container = nmContext.getContainers().get(containerId);
        tempIsRunning = (container.getContainerState() == ContainerState.RUNNING);
    } catch (Exception ex) {
        // assume the container has already finished.
        if (LOG.isDebugEnabled()) {
            LOG.debug("Can not find the container:" + containerId + " in this node.");
        }
    }
    final boolean isRunning = tempIsRunning;
    File logFile = null;
    try {
        logFile = ContainerLogsUtils.getContainerLogFile(containerId, filename, request.getRemoteUser(), nmContext);
    } catch (NotFoundException ex) {
        if (redirectWSUrl == null || redirectWSUrl.isEmpty()) {
            return Response.status(Status.NOT_FOUND).entity(ex.getMessage()).build();
        }
        // redirect the request to the configured log server
        String redirectURI = "/containers/" + containerIdStr + "/logs/" + filename;
        return createRedirectResponse(request, redirectWSUrl, redirectURI);
    } catch (YarnException ex) {
        return Response.serverError().entity(ex.getMessage()).build();
    }
    final long bytes = parseLongParam(size);
    final String lastModifiedTime = Times.format(logFile.lastModified());
    final String outputFileName = filename;
    String contentType = WebAppUtils.getDefaultLogContentType();
    if (format != null && !format.isEmpty()) {
        contentType = WebAppUtils.getSupportedLogContentType(format);
        if (contentType == null) {
            String errorMessage = "The valid values for the parameter : format " + "are " + WebAppUtils.listSupportedLogContentType();
            return Response.status(Status.BAD_REQUEST).entity(errorMessage).build();
        }
    }
    try {
        final FileInputStream fis = ContainerLogsUtils.openLogFileForRead(containerIdStr, logFile, nmContext);
        final long fileLength = logFile.length();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream os) throws IOException, WebApplicationException {
                try {
                    int bufferSize = 65536;
                    byte[] buf = new byte[bufferSize];
                    LogToolUtils.outputContainerLog(containerId.toString(), nmContext.getNodeId().toString(), outputFileName, fileLength, bytes, lastModifiedTime, fis, os, buf, ContainerLogAggregationType.LOCAL);
                    StringBuilder sb = new StringBuilder();
                    String endOfFile = "End of LogType:" + outputFileName;
                    sb.append(endOfFile + ".");
                    if (isRunning) {
                        sb.append("This log file belongs to a running container (" + containerIdStr + ") and so may not be complete." + "\n");
                    } else {
                        sb.append("\n");
                    }
                    sb.append(StringUtils.repeat("*", endOfFile.length() + 50) + "\n\n");
                    os.write(sb.toString().getBytes(Charset.forName("UTF-8")));
                    // If we have aggregated logs for this container,
                    // output the aggregation logs as well.
                    ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
                    Application app = nmContext.getApplications().get(appId);
                    String appOwner = app == null ? null : app.getUser();
                    try {
                        LogToolUtils.outputAggregatedContainerLog(nmContext.getConf(), appId, appOwner, containerId.toString(), nmContext.getNodeId().toString(), outputFileName, bytes, os, buf);
                    } catch (Exception ex) {
                        // Something wrong when we try to access the aggregated log.
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Can not access the aggregated log for " + "the container:" + containerId);
                            LOG.debug(ex.getMessage());
                        }
                    }
                } finally {
                    IOUtils.closeQuietly(fis);
                }
            }
        };
        ResponseBuilder resp = Response.ok(stream);
        resp.header("Content-Type", contentType + "; " + JettyUtils.UTF_8);
        // Sending the X-Content-Type-Options response header with the value
        // nosniff will prevent Internet Explorer from MIME-sniffing a response
        // away from the declared content-type.
        resp.header("X-Content-Type-Options", "nosniff");
        return resp.build();
    } catch (IOException ex) {
        return Response.serverError().entity(ex.getMessage()).build();
    }
}

Example 2 with Unstable

use of org.apache.hadoop.classification.InterfaceStability.Unstable in project hadoop by apache.

the class AHSWebServices method getLogs.

//TODO: YARN-4993: Refactory ContainersLogsBlock, AggregatedLogsBlock and
//      container log webservice introduced in AHS to minimize
//      the duplication.
@GET
@Path("/containerlogs/{containerid}/{filename}")
@Produces({ MediaType.TEXT_PLAIN + "; " + JettyUtils.UTF_8 })
@Public
@Unstable
public Response getLogs(@Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size, @QueryParam(YarnWebServiceParams.NM_ID) String nmId, @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) @DefaultValue("false") boolean redirected_from_node) {
    init(res);
    ContainerId containerId;
    try {
        containerId = ContainerId.fromString(containerIdStr);
    } catch (IllegalArgumentException ex) {
        return createBadResponse(Status.NOT_FOUND, "Invalid ContainerId: " + containerIdStr);
    }
    final long length = parseLongParam(size);
    ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
    AppInfo appInfo;
    try {
        appInfo = super.getApp(req, res, appId.toString());
    } catch (Exception ex) {
        // directly find logs from HDFS.
        return sendStreamOutputResponse(appId, null, null, containerIdStr, filename, format, length, false);
    }
    String appOwner = appInfo.getUser();
    if (isFinishedState(appInfo.getAppState())) {
        // directly find logs from HDFS.
        return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, filename, format, length, false);
    }
    if (isRunningState(appInfo.getAppState())) {
        String nodeHttpAddress = null;
        if (nmId != null && !nmId.isEmpty()) {
            try {
                nodeHttpAddress = getNMWebAddressFromRM(conf, nmId);
            } catch (Exception ex) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(ex.getMessage());
                }
            }
        }
        if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
            ContainerInfo containerInfo;
            try {
                containerInfo = super.getContainer(req, res, appId.toString(), containerId.getApplicationAttemptId().toString(), containerId.toString());
            } catch (Exception ex) {
                // output the aggregated logs
                return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, filename, format, length, true);
            }
            nodeHttpAddress = containerInfo.getNodeHttpAddress();
            // request back. Simply output the aggregated logs.
            if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() || redirected_from_node) {
                // output the aggregated logs
                return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, filename, format, length, true);
            }
        }
        String uri = "/" + containerId.toString() + "/logs/" + filename;
        String resURI = JOINER.join(getAbsoluteNMWebAddress(nodeHttpAddress), NM_DOWNLOAD_URI_STR, uri);
        String query = req.getQueryString();
        if (query != null && !query.isEmpty()) {
            resURI += "?" + query;
        }
        ResponseBuilder response = Response.status(HttpServletResponse.SC_TEMPORARY_REDIRECT);
        response.header("Location", resURI);
        return response.build();
    } else {
        return createBadResponse(Status.NOT_FOUND, "The application is not at Running or Finished State.");
    }
}

Example 3 with Unstable

use of org.apache.hadoop.classification.InterfaceStability.Unstable in project hadoop by apache.

the class ZKRMStateStore method constructZkRootNodeACL.

/**
   * Given the {@link Configuration} and {@link ACL}s used (sourceACLs) for
   * ZooKeeper access, construct the {@link ACL}s for the store's root node.
   * In the constructed {@link ACL}, all the users allowed by sourceACLs are
   * given read-write-admin access, while the current RM has exclusive
   * create-delete access.
   *
   * To be called only when HA is enabled and the configuration doesn't set an
   * ACL for the root node.
   * @param conf the configuration
   * @param sourceACLs the source ACLs
   * @return ACLs for the store's root node
   * @throws java.security.NoSuchAlgorithmException thrown if the digest
   * algorithm used by Zookeeper cannot be found
   */
@VisibleForTesting
@Private
@Unstable
protected List<ACL> constructZkRootNodeACL(Configuration conf, List<ACL> sourceACLs) throws NoSuchAlgorithmException {
    List<ACL> zkRootNodeAclList = new ArrayList<>();
    for (ACL acl : sourceACLs) {
        zkRootNodeAclList.add(new ACL(ZKUtil.removeSpecificPerms(acl.getPerms(), CREATE_DELETE_PERMS), acl.getId()));
    }
    zkRootNodeUsername = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, conf);
    Id rmId = new Id(zkRootNodeAuthScheme, DigestAuthenticationProvider.generateDigest(zkRootNodeUsername + ":" + resourceManager.getZkRootNodePassword()));
    zkRootNodeAclList.add(new ACL(CREATE_DELETE_PERMS, rmId));
    return zkRootNodeAclList;
}

Example 4 with Unstable

use of org.apache.hadoop.classification.InterfaceStability.Unstable in project tez by apache.

the class TezClient method appIdfromString.

// Copied this helper method from
// org.apache.hadoop.yarn.api.records.ApplicationId in Hadoop 2.8+
// to simplify implementation on 2.7.x
@Public
@Unstable
public static ApplicationId appIdfromString(String appIdStr) {
    if (!appIdStr.startsWith(APPLICATION_ID_PREFIX)) {
        throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + appIdStrPrefix);
    }
    try {
        int pos1 = APPLICATION_ID_PREFIX.length() - 1;
        int pos2 = appIdStr.indexOf('_', pos1 + 1);
        if (pos2 < 0) {
            throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr);
        }
        long rmId = Long.parseLong(appIdStr.substring(pos1 + 1, pos2));
        int appId = Integer.parseInt(appIdStr.substring(pos2 + 1));
        ApplicationId applicationId = ApplicationId.newInstance(rmId, appId);
        return applicationId;
    } catch (NumberFormatException n) {
        throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr, n);
    }
}

Example 5 with Unstable

use of org.apache.hadoop.classification.InterfaceStability.Unstable in project hadoop by apache.

the class ClientRMProxy method getTokenService.

@Unstable
public static Text getTokenService(Configuration conf, String address, String defaultAddr, int defaultPort) {
    if (HAUtil.isHAEnabled(conf)) {
        // Build a list of service addresses to form the service name
        ArrayList<String> services = new ArrayList<String>();
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        for (String rmId : HAUtil.getRMHAIds(conf)) {
            // Set RM_ID to get the corresponding RM_ADDRESS
            yarnConf.set(YarnConfiguration.RM_HA_ID, rmId);
            services.add(SecurityUtil.buildTokenService(yarnConf.getSocketAddr(address, defaultAddr, defaultPort)).toString());
        }
        return new Text(Joiner.on(',').join(services));
    }
    // Non-HA case - no need to set RM_ID
    return SecurityUtil.buildTokenService(conf.getSocketAddr(address, defaultAddr, defaultPort));
}