Examples with YarnException org.apache.hadoop.yarn.exceptions.YarnException used on opensource projects

Example 91 with YarnException

use of org.apache.hadoop.yarn.exceptions.YarnException in project hadoop by apache.

the class ClientServiceDelegate method invoke.

private synchronized Object invoke(String method, Class argClass, Object args) throws IOException {
    Method methodOb = null;
    try {
        methodOb = MRClientProtocol.class.getMethod(method, argClass);
    } catch (SecurityException e) {
        throw new YarnRuntimeException(e);
    } catch (NoSuchMethodException e) {
        throw new YarnRuntimeException("Method name mismatch", e);
    }
    maxClientRetry = this.conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES);
    IOException lastException = null;
    while (maxClientRetry > 0) {
        MRClientProtocol MRClientProxy = null;
        try {
            MRClientProxy = getProxy();
            return methodOb.invoke(MRClientProxy, args);
        } catch (InvocationTargetException e) {
            // Will not throw out YarnException anymore
            LOG.debug("Failed to contact AM/History for job " + jobId + " retrying..", e.getTargetException());
            // Force reconnection by setting the proxy to null.
            realProxy = null;
            if (e.getCause() instanceof AuthorizationException) {
                throw new IOException(e.getTargetException());
            }
            // for its AM to be restarted.
            if (!usingAMProxy.get()) {
                maxClientRetry--;
            }
            usingAMProxy.set(false);
            lastException = new IOException(e.getTargetException());
            try {
                Thread.sleep(100);
            } catch (InterruptedException ie) {
                LOG.warn("ClientServiceDelegate invoke call interrupted", ie);
                throw new YarnRuntimeException(ie);
            }
        } catch (Exception e) {
            LOG.debug("Failed to contact AM/History for job " + jobId + "  Will retry..", e);
            // Force reconnection by setting the proxy to null.
            realProxy = null;
            // RM shutdown
            maxClientRetry--;
            lastException = new IOException(e.getMessage());
            try {
                Thread.sleep(100);
            } catch (InterruptedException ie) {
                LOG.warn("ClientServiceDelegate invoke call interrupted", ie);
                throw new YarnRuntimeException(ie);
            }
        }
    }
    throw lastException;
}

Example 92 with YarnException

use of org.apache.hadoop.yarn.exceptions.YarnException in project hadoop by apache.

the class ClientServiceDelegate method getProxy.

private MRClientProtocol getProxy() throws IOException {
    if (realProxy != null) {
        return realProxy;
    }
    // Possibly allow nulls through the PB tunnel, otherwise deal with an exception
    // and redirect to the history server.
    ApplicationReport application = null;
    try {
        application = rm.getApplicationReport(appId);
    } catch (ApplicationNotFoundException e) {
        application = null;
    } catch (YarnException e2) {
        throw new IOException(e2);
    }
    if (application != null) {
        trackingUrl = application.getTrackingUrl();
    }
    InetSocketAddress serviceAddr = null;
    while (application == null || YarnApplicationState.RUNNING == application.getYarnApplicationState()) {
        if (application == null) {
            LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server.");
            return checkAndGetHSProxy(null, JobState.NEW);
        }
        try {
            if (application.getHost() == null || "".equals(application.getHost())) {
                LOG.debug("AM not assigned to Job. Waiting to get the AM ...");
                Thread.sleep(2000);
                LOG.debug("Application state is " + application.getYarnApplicationState());
                application = rm.getApplicationReport(appId);
                continue;
            } else if (UNAVAILABLE.equals(application.getHost())) {
                if (!amAclDisabledStatusLogged) {
                    LOG.info("Job " + jobId + " is running, but the host is unknown." + " Verify user has VIEW_JOB access.");
                    amAclDisabledStatusLogged = true;
                }
                return getNotRunningJob(application, JobState.RUNNING);
            }
            if (!conf.getBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, false)) {
                UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName());
                serviceAddr = NetUtils.createSocketAddrForHost(application.getHost(), application.getRpcPort());
                if (UserGroupInformation.isSecurityEnabled()) {
                    org.apache.hadoop.yarn.api.records.Token clientToAMToken = application.getClientToAMToken();
                    Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr);
                    newUgi.addToken(token);
                }
                LOG.debug("Connecting to " + serviceAddr);
                final InetSocketAddress finalServiceAddr = serviceAddr;
                realProxy = newUgi.doAs(new PrivilegedExceptionAction<MRClientProtocol>() {

                    @Override
                    public MRClientProtocol run() throws IOException {
                        return instantiateAMProxy(finalServiceAddr);
                    }
                });
            } else {
                if (!amAclDisabledStatusLogged) {
                    LOG.info("Network ACL closed to AM for job " + jobId + ". Not going to try to reach the AM.");
                    amAclDisabledStatusLogged = true;
                }
                return getNotRunningJob(null, JobState.RUNNING);
            }
            return realProxy;
        } catch (IOException e) {
            //possibly the AM has crashed
            //there may be some time before AM is restarted
            //keep retrying by getting the address from RM
            LOG.info("Could not connect to " + serviceAddr + ". Waiting for getting the latest AM address...");
            try {
                Thread.sleep(2000);
            } catch (InterruptedException e1) {
                LOG.warn("getProxy() call interruped", e1);
                throw new YarnRuntimeException(e1);
            }
            try {
                application = rm.getApplicationReport(appId);
            } catch (YarnException e1) {
                throw new IOException(e1);
            }
            if (application == null) {
                LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server.");
                return checkAndGetHSProxy(null, JobState.RUNNING);
            }
        } catch (InterruptedException e) {
            LOG.warn("getProxy() call interruped", e);
            throw new YarnRuntimeException(e);
        } catch (YarnException e) {
            throw new IOException(e);
        }
    }
    /** we just want to return if its allocating, so that we don't
     * block on it. This is to be able to return job status
     * on an allocating Application.
     */
    String user = application.getUser();
    if (user == null) {
        throw new IOException("User is not set in the application report");
    }
    if (application.getYarnApplicationState() == YarnApplicationState.NEW || application.getYarnApplicationState() == YarnApplicationState.NEW_SAVING || application.getYarnApplicationState() == YarnApplicationState.SUBMITTED || application.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.NEW);
    }
    if (application.getYarnApplicationState() == YarnApplicationState.FAILED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.FAILED);
    }
    if (application.getYarnApplicationState() == YarnApplicationState.KILLED) {
        realProxy = null;
        return getNotRunningJob(application, JobState.KILLED);
    }
    //succeeded.
    if (application.getYarnApplicationState() == YarnApplicationState.FINISHED) {
        LOG.info("Application state is completed. FinalApplicationStatus=" + application.getFinalApplicationStatus().toString() + ". Redirecting to job history server");
        realProxy = checkAndGetHSProxy(application, JobState.SUCCEEDED);
    }
    return realProxy;
}

Example 93 with YarnException

use of org.apache.hadoop.yarn.exceptions.YarnException in project hadoop by apache.

the class ResourceMgrDelegate method getClusterMetrics.

public ClusterMetrics getClusterMetrics() throws IOException, InterruptedException {
    try {
        YarnClusterMetrics metrics = client.getYarnClusterMetrics();
        ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1, metrics.getNumNodeManagers(), 0, 0);
        return oldMetrics;
    } catch (YarnException e) {
        throw new IOException(e);
    }
}

Example 94 with YarnException

use of org.apache.hadoop.yarn.exceptions.YarnException in project hadoop by apache.

the class ResourceMgrDelegate method getAllJobs.

public JobStatus[] getAllJobs() throws IOException, InterruptedException {
    try {
        Set<String> appTypes = new HashSet<String>(1);
        appTypes.add(MRJobConfig.MR_APPLICATION_TYPE);
        EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class);
        return TypeConverter.fromYarnApps(client.getApplications(appTypes, appStates), this.conf);
    } catch (YarnException e) {
        throw new IOException(e);
    }
}

Example 95 with YarnException

use of org.apache.hadoop.yarn.exceptions.YarnException in project hadoop by apache.

the class YARNRunner method killUnFinishedApplication.

private void killUnFinishedApplication(ApplicationId appId) throws IOException {
    ApplicationReport application = null;
    try {
        application = resMgrDelegate.getApplicationReport(appId);
    } catch (YarnException e) {
        throw new IOException(e);
    }
    if (application.getYarnApplicationState() == YarnApplicationState.FINISHED || application.getYarnApplicationState() == YarnApplicationState.FAILED || application.getYarnApplicationState() == YarnApplicationState.KILLED) {
        return;
    }
    killApplication(appId);
}