Examples with YarnRuntimeException org.apache.hadoop.yarn.exceptions.YarnRuntimeException used on opensource projects

Example 21 with YarnRuntimeException

use of org.apache.hadoop.yarn.exceptions.YarnRuntimeException in project hadoop by apache.

the class RMWebAppFilter method ahsRedirectPath.

private String ahsRedirectPath(String uri, RMWebApp rmWebApp) {
    // TODO: Commonize URL parsing code. Will be done in YARN-4642.
    String redirectPath = null;
    if (uri.contains("/cluster/")) {
        String[] parts = uri.split("/");
        if (parts.length > 3) {
            RMContext context = rmWebApp.getRMContext();
            String type = parts[2];
            ApplicationId appId = null;
            ApplicationAttemptId appAttemptId = null;
            ContainerId containerId = null;
            switch(type) {
                case "app":
                    try {
                        appId = Apps.toAppID(parts[3]);
                    } catch (YarnRuntimeException | NumberFormatException e) {
                        LOG.debug("Error parsing {} as an ApplicationId", parts[3], e);
                        return redirectPath;
                    }
                    if (!context.getRMApps().containsKey(appId)) {
                        redirectPath = pjoin(ahsPageURLPrefix, "app", appId);
                    }
                    break;
                case "appattempt":
                    try {
                        appAttemptId = ApplicationAttemptId.fromString(parts[3]);
                    } catch (IllegalArgumentException e) {
                        LOG.debug("Error parsing {} as an ApplicationAttemptId", parts[3], e);
                        return redirectPath;
                    }
                    if (!context.getRMApps().containsKey(appAttemptId.getApplicationId())) {
                        redirectPath = pjoin(ahsPageURLPrefix, "appattempt", appAttemptId);
                    }
                    break;
                case "container":
                    try {
                        containerId = ContainerId.fromString(parts[3]);
                    } catch (IllegalArgumentException e) {
                        LOG.debug("Error parsing {} as an ContainerId", parts[3], e);
                        return redirectPath;
                    }
                    if (!context.getRMApps().containsKey(containerId.getApplicationAttemptId().getApplicationId())) {
                        redirectPath = pjoin(ahsPageURLPrefix, "container", containerId);
                    }
                    break;
                default:
                    break;
            }
        }
    }
    return redirectPath;
}

Example 22 with YarnRuntimeException

use of org.apache.hadoop.yarn.exceptions.YarnRuntimeException in project hadoop by apache.

the class RMWebServices method getApps.

@GET
@Path("/apps")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public AppsInfo getApps(@Context HttpServletRequest hsr, @QueryParam("state") String stateQuery, @QueryParam("states") Set<String> statesQuery, @QueryParam("finalStatus") String finalStatusQuery, @QueryParam("user") String userQuery, @QueryParam("queue") String queueQuery, @QueryParam("limit") String count, @QueryParam("startedTimeBegin") String startedBegin, @QueryParam("startedTimeEnd") String startedEnd, @QueryParam("finishedTimeBegin") String finishBegin, @QueryParam("finishedTimeEnd") String finishEnd, @QueryParam("applicationTypes") Set<String> applicationTypes, @QueryParam("applicationTags") Set<String> applicationTags) {
    boolean checkCount = false;
    boolean checkStart = false;
    boolean checkEnd = false;
    boolean checkAppTypes = false;
    boolean checkAppStates = false;
    boolean checkAppTags = false;
    long countNum = 0;
    // set values suitable in case both of begin/end not specified
    long sBegin = 0;
    long sEnd = Long.MAX_VALUE;
    long fBegin = 0;
    long fEnd = Long.MAX_VALUE;
    init();
    if (count != null && !count.isEmpty()) {
        checkCount = true;
        countNum = Long.parseLong(count);
        if (countNum <= 0) {
            throw new BadRequestException("limit value must be greater then 0");
        }
    }
    if (startedBegin != null && !startedBegin.isEmpty()) {
        checkStart = true;
        sBegin = Long.parseLong(startedBegin);
        if (sBegin < 0) {
            throw new BadRequestException("startedTimeBegin must be greater than 0");
        }
    }
    if (startedEnd != null && !startedEnd.isEmpty()) {
        checkStart = true;
        sEnd = Long.parseLong(startedEnd);
        if (sEnd < 0) {
            throw new BadRequestException("startedTimeEnd must be greater than 0");
        }
    }
    if (sBegin > sEnd) {
        throw new BadRequestException("startedTimeEnd must be greater than startTimeBegin");
    }
    if (finishBegin != null && !finishBegin.isEmpty()) {
        checkEnd = true;
        fBegin = Long.parseLong(finishBegin);
        if (fBegin < 0) {
            throw new BadRequestException("finishTimeBegin must be greater than 0");
        }
    }
    if (finishEnd != null && !finishEnd.isEmpty()) {
        checkEnd = true;
        fEnd = Long.parseLong(finishEnd);
        if (fEnd < 0) {
            throw new BadRequestException("finishTimeEnd must be greater than 0");
        }
    }
    if (fBegin > fEnd) {
        throw new BadRequestException("finishTimeEnd must be greater than finishTimeBegin");
    }
    Set<String> appTypes = parseQueries(applicationTypes, false);
    if (!appTypes.isEmpty()) {
        checkAppTypes = true;
    }
    Set<String> appTags = parseQueries(applicationTags, false);
    if (!appTags.isEmpty()) {
        checkAppTags = true;
    }
    // stateQuery is deprecated.
    if (stateQuery != null && !stateQuery.isEmpty()) {
        statesQuery.add(stateQuery);
    }
    Set<String> appStates = parseQueries(statesQuery, true);
    if (!appStates.isEmpty()) {
        checkAppStates = true;
    }
    GetApplicationsRequest request = GetApplicationsRequest.newInstance();
    if (checkStart) {
        request.setStartRange(sBegin, sEnd);
    }
    if (checkEnd) {
        request.setFinishRange(fBegin, fEnd);
    }
    if (checkCount) {
        request.setLimit(countNum);
    }
    if (checkAppTypes) {
        request.setApplicationTypes(appTypes);
    }
    if (checkAppTags) {
        request.setApplicationTags(appTags);
    }
    if (checkAppStates) {
        request.setApplicationStates(appStates);
    }
    if (queueQuery != null && !queueQuery.isEmpty()) {
        ResourceScheduler rs = rm.getResourceScheduler();
        if (rs instanceof CapacityScheduler) {
            CapacityScheduler cs = (CapacityScheduler) rs;
            // validate queue exists
            try {
                cs.getQueueInfo(queueQuery, false, false);
            } catch (IOException e) {
                throw new BadRequestException(e.getMessage());
            }
        }
        Set<String> queues = new HashSet<String>(1);
        queues.add(queueQuery);
        request.setQueues(queues);
    }
    if (userQuery != null && !userQuery.isEmpty()) {
        Set<String> users = new HashSet<String>(1);
        users.add(userQuery);
        request.setUsers(users);
    }
    List<ApplicationReport> appReports = null;
    try {
        appReports = rm.getClientRMService().getApplications(request, false).getApplicationList();
    } catch (YarnException e) {
        LOG.error("Unable to retrieve apps from ClientRMService", e);
        throw new YarnRuntimeException("Unable to retrieve apps from ClientRMService", e);
    }
    final ConcurrentMap<ApplicationId, RMApp> apps = rm.getRMContext().getRMApps();
    AppsInfo allApps = new AppsInfo();
    for (ApplicationReport report : appReports) {
        RMApp rmapp = apps.get(report.getApplicationId());
        if (rmapp == null) {
            continue;
        }
        if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) {
            FinalApplicationStatus.valueOf(finalStatusQuery);
            if (!rmapp.getFinalApplicationStatus().toString().equalsIgnoreCase(finalStatusQuery)) {
                continue;
            }
        }
        AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr), WebAppUtils.getHttpSchemePrefix(conf));
        allApps.add(app);
    }
    return allApps;
}

Example 23 with YarnRuntimeException

use of org.apache.hadoop.yarn.exceptions.YarnRuntimeException in project hadoop by apache.

the class RMWebServices method createNewApplication.

/**
   * Function that actually creates the ApplicationId by calling the
   * ClientRMService
   * 
   * @return returns structure containing the app-id and maximum resource
   *         capabilities
   */
private NewApplication createNewApplication() {
    GetNewApplicationRequest req = recordFactory.newRecordInstance(GetNewApplicationRequest.class);
    GetNewApplicationResponse resp;
    try {
        resp = rm.getClientRMService().getNewApplication(req);
    } catch (YarnException e) {
        String msg = "Unable to create new app from RM web service";
        LOG.error(msg, e);
        throw new YarnRuntimeException(msg, e);
    }
    NewApplication appId = new NewApplication(resp.getApplicationId().toString(), new ResourceInfo(resp.getMaximumResourceCapability()));
    return appId;
}

Example 24 with YarnRuntimeException

use of org.apache.hadoop.yarn.exceptions.YarnRuntimeException in project hadoop by apache.

the class ClientServiceDelegate method invoke.

private synchronized Object invoke(String method, Class argClass, Object args) throws IOException {
    Method methodOb = null;
    try {
        methodOb = MRClientProtocol.class.getMethod(method, argClass);
    } catch (SecurityException e) {
        throw new YarnRuntimeException(e);
    } catch (NoSuchMethodException e) {
        throw new YarnRuntimeException("Method name mismatch", e);
    }
    maxClientRetry = this.conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES);
    IOException lastException = null;
    while (maxClientRetry > 0) {
        MRClientProtocol MRClientProxy = null;
        try {
            MRClientProxy = getProxy();
            return methodOb.invoke(MRClientProxy, args);
        } catch (InvocationTargetException e) {
            // Will not throw out YarnException anymore
            LOG.debug("Failed to contact AM/History for job " + jobId + " retrying..", e.getTargetException());
            // Force reconnection by setting the proxy to null.
            realProxy = null;
            if (e.getCause() instanceof AuthorizationException) {
                throw new IOException(e.getTargetException());
            }
            // for its AM to be restarted.
            if (!usingAMProxy.get()) {
                maxClientRetry--;
            }
            usingAMProxy.set(false);
            lastException = new IOException(e.getTargetException());
            try {
                Thread.sleep(100);
            } catch (InterruptedException ie) {
                LOG.warn("ClientServiceDelegate invoke call interrupted", ie);
                throw new YarnRuntimeException(ie);
            }
        } catch (Exception e) {
            LOG.debug("Failed to contact AM/History for job " + jobId + "  Will retry..", e);
            // Force reconnection by setting the proxy to null.
            realProxy = null;
            // RM shutdown
            maxClientRetry--;
            lastException = new IOException(e.getMessage());
            try {
                Thread.sleep(100);
            } catch (InterruptedException ie) {
                LOG.warn("ClientServiceDelegate invoke call interrupted", ie);
                throw new YarnRuntimeException(ie);
            }
        }
    }
    throw lastException;
}

Example 25 with YarnRuntimeException

use of org.apache.hadoop.yarn.exceptions.YarnRuntimeException in project hadoop by apache.

the class ClientServiceDelegate method getProxy.

private MRClientProtocol getProxy() throws IOException {
    if (realProxy != null) {
        return realProxy;
    }
    // Possibly allow nulls through the PB tunnel, otherwise deal with an exception
    // and redirect to the history server.
    ApplicationReport application = null;
    try {
        application = rm.getApplicationReport(appId);
    } catch (ApplicationNotFoundException e) {
        application = null;
    } catch (YarnException e2) {
        throw new IOException(e2);
    }
    if (application != null) {
        trackingUrl = application.getTrackingUrl();
    }
    InetSocketAddress serviceAddr = null;
    while (application == null || YarnApplicationState.RUNNING == application.getYarnApplicationState()) {
        if (application == null) {
            LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server.");
            return checkAndGetHSProxy(null, JobState.NEW);
        }
        try {
            if (application.getHost() == null || "".equals(application.getHost())) {
                LOG.debug("AM not assigned to Job. Waiting to get the AM ...");
                Thread.sleep(2000);
                LOG.debug("Application state is " + application.getYarnApplicationState());
                application = rm.getApplicationReport(appId);
                continue;
            } else if (UNAVAILABLE.equals(application.getHost())) {
                if (!amAclDisabledStatusLogged) {
                    LOG.info("Job " + jobId + " is running, but the host is unknown." + " Verify user has VIEW_JOB access.");
                    amAclDisabledStatusLogged = true;
                }
                return getNotRunningJob(application, JobState.RUNNING);
            }
            if (!conf.getBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, false)) {
                UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName());
                serviceAddr = NetUtils.createSocketAddrForHost(application.getHost(), application.getRpcPort());
                if (UserGroupInformation.isSecurityEnabled()) {
                    org.apache.hadoop.yarn.api.records.Token clientToAMToken = application.getClientToAMToken();
                    Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr);
                    newUgi.addToken(token);
                }
                LOG.debug("Connecting to " + serviceAddr);
                final InetSocketAddress finalServiceAddr = serviceAddr;
                realProxy = newUgi.doAs(new PrivilegedExceptionAction<MRClientProtocol>() {

                    @Override
                    public MRClientProtocol run() throws IOException {
                        return instantiateAMProxy(finalServiceAddr);
                    }
                });
            } else {
                if (!amAclDisabledStatusLogged) {
                    LOG.info("Network ACL closed to AM for job " + jobId + ". Not going to try to reach the AM.");
                    amAclDisabledStatusLogged = true;
                }
                return getNotRunningJob(null, JobState.RUNNING);
            }
            return realProxy;
        } catch (IOException e) {
            //possibly the AM has crashed
            //there may be some time before AM is restarted
            //keep retrying by getting the address from RM
            LOG.info("Could not connect to " + serviceAddr + ". Waiting for getting the latest AM address...");
            try {
                Thread.sleep(2000);
            } catch (InterruptedException e1) {
                LOG.warn("getProxy() call interruped", e1);
                throw new YarnRuntimeException(e1);
            }
            try {
                application = rm.getApplicationReport(appId);
            } catch (YarnException e1) {
                throw new IOException(e1);
            }
            if (application == null) {
                LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server.");
                return checkAndGetHSProxy(null, JobState.RUNNING);
            }
        } catch (InterruptedException e) {
            LOG.warn("getProxy() call interruped", e);
            throw new YarnRuntimeException(e);
        } catch (YarnException e) {
            throw new IOException(e);
        }
    }
    /** we just want to return if its allocating, so that we don't
     * block on it. This is to be able to return job status
     * on an allocating Application.
     */
    String user = application.getUser();
    if (user == null) {
        throw new IOException("User is not set in the application report");
    }
    if (application.getYarnApplicationState() == YarnApplicationState.NEW || application.getYarnApplicationState() == YarnApplicationState.NEW_SAVING || application.getYarnApplicationState() == YarnApplicationState.SUBMITTED || application.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.NEW);
    }
    if (application.getYarnApplicationState() == YarnApplicationState.FAILED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.FAILED);
    }
    if (application.getYarnApplicationState() == YarnApplicationState.KILLED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.KILLED);
    }
    //succeeded.
    if (application.getYarnApplicationState() == YarnApplicationState.FINISHED) {
        LOG.info("Application state is completed. FinalApplicationStatus=" + application.getFinalApplicationStatus().toString() + ". Redirecting to job history server");
        realProxy = checkAndGetHSProxy(application, JobState.SUCCEEDED);
    }
    return realProxy;
}