Examples with ActionExecutorException org.apache.oozie.action.ActionExecutorException used on opensource projects

Example 41 with ActionExecutorException

use of org.apache.oozie.action.ActionExecutorException in project oozie by apache.

the class JavaActionExecutor method addShareLib.

protected void addShareLib(Configuration conf, String[] actionShareLibNames) throws ActionExecutorException {
    Set<String> confSet = new HashSet<String>(Arrays.asList(getShareLibFilesForActionConf() == null ? new String[0] : getShareLibFilesForActionConf()));
    Set<Path> sharelibList = new HashSet<Path>();
    if (actionShareLibNames != null) {
        try {
            ShareLibService shareLibService = Services.get().get(ShareLibService.class);
            FileSystem fs = shareLibService.getFileSystem();
            if (fs != null) {
                for (String actionShareLibName : actionShareLibNames) {
                    List<Path> listOfPaths = shareLibService.getShareLibJars(actionShareLibName);
                    if (listOfPaths != null && !listOfPaths.isEmpty()) {
                        for (Path actionLibPath : listOfPaths) {
                            String fragmentName = new URI(actionLibPath.toString()).getFragment();
                            String fileName = fragmentName == null ? actionLibPath.getName() : fragmentName;
                            if (confSet.contains(fileName)) {
                                Configuration jobXmlConf = shareLibService.getShareLibConf(actionShareLibName, actionLibPath);
                                if (jobXmlConf != null) {
                                    checkForDisallowedProps(jobXmlConf, actionLibPath.getName());
                                    XConfiguration.injectDefaults(jobXmlConf, conf);
                                    LOG.trace("Adding properties of " + actionLibPath + " to job conf");
                                }
                            } else {
                                // Filtering out duplicate jars or files
                                sharelibList.add(new Path(actionLibPath.toUri()) {

                                    @Override
                                    public int hashCode() {
                                        return getName().hashCode();
                                    }

                                    @Override
                                    public String getName() {
                                        try {
                                            return (new URI(toString())).getFragment() == null ? new Path(toUri()).getName() : (new URI(toString())).getFragment();
                                        } catch (URISyntaxException e) {
                                            throw new RuntimeException(e);
                                        }
                                    }

                                    @Override
                                    public boolean equals(Object input) {
                                        if (input == null) {
                                            return false;
                                        }
                                        if (input == this) {
                                            return true;
                                        }
                                        if (!(input instanceof Path)) {
                                            return false;
                                        }
                                        return getName().equals(((Path) input).getName());
                                    }
                                });
                            }
                        }
                    }
                }
            }
            addLibPathsToCache(conf, sharelibList);
        } catch (URISyntaxException ex) {
            throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "Error configuring sharelib", ex.getMessage());
        } catch (IOException ex) {
            throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "It should never happen", ex.getMessage());
        }
    }
}

Example 42 with ActionExecutorException

use of org.apache.oozie.action.ActionExecutorException in project oozie by apache.

the class JavaActionExecutor method addToCache.

Configuration addToCache(Configuration conf, Path appPath, String filePath, boolean archive) throws ActionExecutorException {
    URI uri = null;
    try {
        uri = new URI(getTrimmedEncodedPath(filePath));
        URI baseUri = appPath.toUri();
        if (uri.getScheme() == null) {
            String resolvedPath = uri.getPath();
            if (!resolvedPath.startsWith("/")) {
                resolvedPath = baseUri.getPath() + "/" + resolvedPath;
            }
            uri = new URI(baseUri.getScheme(), baseUri.getAuthority(), resolvedPath, uri.getQuery(), uri.getFragment());
        }
        if (archive) {
            DistributedCache.addCacheArchive(uri.normalize(), conf);
        } else {
            String fileName = filePath.substring(filePath.lastIndexOf("/") + 1);
            if (fileName.endsWith(".so") || fileName.contains(".so.")) {
                // .so files
                uri = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), uri.getQuery(), fileName);
                DistributedCache.addCacheFile(uri.normalize(), conf);
            } else if (fileName.endsWith(".jar")) {
                // .jar files
                if (!fileName.contains("#")) {
                    String user = conf.get("user.name");
                    if (FSUtils.isNotLocalFile(fileName)) {
                        Path pathToAdd = new Path(uri.normalize());
                        Services.get().get(HadoopAccessorService.class).addFileToClassPath(user, pathToAdd, conf);
                    }
                } else {
                    DistributedCache.addCacheFile(uri.normalize(), conf);
                }
            } else {
                // regular files
                if (!fileName.contains("#")) {
                    uri = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), uri.getQuery(), fileName);
                }
                DistributedCache.addCacheFile(uri.normalize(), conf);
            }
        }
        DistributedCache.createSymlink(conf);
        return conf;
    } catch (Exception ex) {
        LOG.debug("Errors when add to DistributedCache. Path=" + Objects.toString(uri, "<null>") + ", archive=" + archive + ", conf=" + XmlUtils.prettyPrint(conf).toString());
        throw convertException(ex);
    }
}

Example 43 with ActionExecutorException

use of org.apache.oozie.action.ActionExecutorException in project oozie by apache.

the class JavaActionExecutor method submitLauncher.

public void submitLauncher(FileSystem actionFs, final Context context, WorkflowAction action) throws ActionExecutorException {
    YarnClient yarnClient = null;
    try {
        Path appPathRoot = new Path(context.getWorkflow().getAppPath());
        // app path could be a file
        if (actionFs.isFile(appPathRoot)) {
            appPathRoot = appPathRoot.getParent();
        }
        Element actionXml = XmlUtils.parseXml(action.getConf());
        LOG.debug("ActionXML: {0}", action.getConf());
        // action job configuration
        Configuration actionConf = loadHadoopDefaultResources(context, actionXml);
        setupActionConf(actionConf, context, actionXml, appPathRoot);
        addAppNameContext(action, context);
        LOG.debug("Setting LibFilesArchives ");
        setLibFilesArchives(context, actionXml, appPathRoot, actionConf);
        String jobName = actionConf.get(HADOOP_JOB_NAME);
        if (jobName == null || jobName.isEmpty()) {
            jobName = XLog.format("oozie:action:T={0}:W={1}:A={2}:ID={3}", getType(), context.getWorkflow().getAppName(), action.getName(), context.getWorkflow().getId());
            actionConf.set(HADOOP_JOB_NAME, jobName);
        }
        injectActionCallback(context, actionConf);
        if (actionConf.get(ACL_MODIFY_JOB) == null || actionConf.get(ACL_MODIFY_JOB).trim().equals("")) {
            // modify-job ACL specifically
            if (context.getWorkflow().getAcl() != null) {
                // setting the group owning the Oozie job to allow anybody in that
                // group to modify the jobs.
                actionConf.set(ACL_MODIFY_JOB, context.getWorkflow().getAcl());
            }
        }
        Credentials credentials = new Credentials();
        Configuration launcherConf = createLauncherConf(actionFs, context, action, actionXml, actionConf);
        yarnClient = createYarnClient(context, launcherConf);
        Map<String, CredentialsProperties> credentialsProperties = setCredentialPropertyToActionConf(context, action, actionConf);
        if (UserGroupInformation.isSecurityEnabled()) {
            addHadoopCredentialPropertiesToActionConf(credentialsProperties);
        }
        // Adding if action need to set more credential tokens
        Configuration credentialsConf = new Configuration(false);
        XConfiguration.copy(actionConf, credentialsConf);
        setCredentialTokens(credentials, credentialsConf, context, action, credentialsProperties);
        // copy back new entries from credentialsConf
        for (Entry<String, String> entry : credentialsConf) {
            if (actionConf.get(entry.getKey()) == null) {
                actionConf.set(entry.getKey(), entry.getValue());
            }
        }
        String consoleUrl;
        String launcherId = LauncherHelper.getRecoveryId(launcherConf, context.getActionDir(), context.getRecoveryId());
        removeHBaseSettingFromOozieDefaultResource(launcherConf);
        removeHBaseSettingFromOozieDefaultResource(actionConf);
        boolean alreadyRunning = launcherId != null;
        // if user-retry is on, always submit new launcher
        boolean isUserRetry = ((WorkflowActionBean) action).isUserRetry();
        LOG.debug("Creating yarnClient for action {0}", action.getId());
        if (alreadyRunning && !isUserRetry) {
            try {
                ApplicationId appId = ConverterUtils.toApplicationId(launcherId);
                ApplicationReport report = yarnClient.getApplicationReport(appId);
                consoleUrl = report.getTrackingUrl();
            } catch (RemoteException e) {
                // caught when the application id does not exist
                LOG.error("Got RemoteException from YARN", e);
                String jobTracker = launcherConf.get(HADOOP_YARN_RM);
                throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "JA017", "unknown job [{0}@{1}], cannot recover", launcherId, jobTracker);
            }
        } else {
            YarnClientApplication newApp = yarnClient.createApplication();
            ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();
            ApplicationSubmissionContext appContext = createAppSubmissionContext(appId, launcherConf, context, actionConf, action.getName(), credentials, actionXml);
            yarnClient.submitApplication(appContext);
            launcherId = appId.toString();
            LOG.debug("After submission get the launcherId [{0}]", launcherId);
            ApplicationReport appReport = yarnClient.getApplicationReport(appId);
            consoleUrl = appReport.getTrackingUrl();
        }
        String jobTracker = launcherConf.get(HADOOP_YARN_RM);
        context.setStartData(launcherId, jobTracker, consoleUrl);
    } catch (Exception ex) {
        throw convertException(ex);
    } finally {
        if (yarnClient != null) {
            Closeables.closeQuietly(yarnClient);
        }
    }
}

Example 44 with ActionExecutorException

use of org.apache.oozie.action.ActionExecutorException in project oozie by apache.

the class JavaActionExecutor method addSystemShareLibForAction.

protected void addSystemShareLibForAction(Configuration conf) throws ActionExecutorException {
    ShareLibService shareLibService = Services.get().get(ShareLibService.class);
    // ShareLibService is null for test cases
    if (shareLibService != null) {
        try {
            List<Path> listOfPaths = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
            if (listOfPaths.isEmpty()) {
                throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "EJ001", "Could not locate Oozie sharelib");
            }
            addLibPathsToClassPath(conf, listOfPaths);
            addLibPathsToClassPath(conf, shareLibService.getSystemLibJars(getType()));
        } catch (IOException ex) {
            throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "It should never happen", ex.getMessage());
        }
    }
}

Example 45 with ActionExecutorException

use of org.apache.oozie.action.ActionExecutorException in project oozie by apache.

the class JavaActionExecutor method check.

@Override
public void check(Context context, WorkflowAction action) throws ActionExecutorException {
    boolean fallback = false;
    LOG = XLog.resetPrefix(LOG);
    LogUtils.setLogInfo(action);
    YarnClient yarnClient = null;
    try {
        Element actionXml = XmlUtils.parseXml(action.getConf());
        Configuration jobConf = createBaseHadoopConf(context, actionXml);
        FileSystem actionFs = context.getAppFileSystem();
        yarnClient = createYarnClient(context, jobConf);
        FinalApplicationStatus appStatus = null;
        try {
            ApplicationReport appReport = yarnClient.getApplicationReport(ConverterUtils.toApplicationId(action.getExternalId()));
            YarnApplicationState appState = appReport.getYarnApplicationState();
            if (appState == YarnApplicationState.FAILED || appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED) {
                appStatus = appReport.getFinalApplicationStatus();
            }
        } catch (Exception ye) {
            LOG.warn("Exception occurred while checking Launcher AM status; will try checking action data file instead ", ye);
            // Fallback to action data file if we can't find the Launcher AM (maybe it got purged)
            fallback = true;
        }
        if (appStatus != null || fallback) {
            Path actionDir = context.getActionDir();
            // load sequence file into object
            Map<String, String> actionData = LauncherHelper.getActionData(actionFs, actionDir, jobConf);
            if (fallback) {
                String finalStatus = actionData.get(LauncherAM.ACTION_DATA_FINAL_STATUS);
                if (finalStatus != null) {
                    appStatus = FinalApplicationStatus.valueOf(finalStatus);
                } else {
                    context.setExecutionData(FAILED, null);
                    throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "JA017", "Unknown hadoop job [{0}] associated with action [{1}] and couldn't determine status from" + " action data.  Failing this action!", action.getExternalId(), action.getId());
                }
            }
            // MapReduce was launched
            String externalID = actionData.get(LauncherAM.ACTION_DATA_NEW_ID);
            if (externalID != null) {
                context.setExternalChildIDs(externalID);
                LOG.info(XLog.STD, "Hadoop Job was launched : [{0}]", externalID);
            }
            // Multiple child IDs - Pig or Hive action
            String externalIDs = actionData.get(LauncherAM.ACTION_DATA_EXTERNAL_CHILD_IDS);
            if (externalIDs != null) {
                context.setExternalChildIDs(externalIDs);
                LOG.info(XLog.STD, "External Child IDs  : [{0}]", externalIDs);
            }
            LOG.info(XLog.STD, "action completed, external ID [{0}]", action.getExternalId());
            context.setExecutionData(appStatus.toString(), null);
            if (appStatus == FinalApplicationStatus.SUCCEEDED) {
                if (getCaptureOutput(action) && LauncherHelper.hasOutputData(actionData)) {
                    context.setExecutionData(SUCCEEDED, PropertiesUtils.stringToProperties(actionData.get(LauncherAM.ACTION_DATA_OUTPUT_PROPS)));
                    LOG.info(XLog.STD, "action produced output");
                } else {
                    context.setExecutionData(SUCCEEDED, null);
                }
                if (LauncherHelper.hasStatsData(actionData)) {
                    context.setExecutionStats(actionData.get(LauncherAM.ACTION_DATA_STATS));
                    LOG.info(XLog.STD, "action produced stats");
                }
                getActionData(actionFs, action, context);
            } else {
                String errorReason;
                if (actionData.containsKey(LauncherAM.ACTION_DATA_ERROR_PROPS)) {
                    Properties props = PropertiesUtils.stringToProperties(actionData.get(LauncherAM.ACTION_DATA_ERROR_PROPS));
                    String errorCode = props.getProperty("error.code");
                    if ("0".equals(errorCode)) {
                        errorCode = "JA018";
                    }
                    if ("-1".equals(errorCode)) {
                        errorCode = "JA019";
                    }
                    errorReason = props.getProperty("error.reason");
                    LOG.warn("Launcher ERROR, reason: {0}", errorReason);
                    String exMsg = props.getProperty("exception.message");
                    String errorInfo = (exMsg != null) ? exMsg : errorReason;
                    context.setErrorInfo(errorCode, errorInfo);
                    String exStackTrace = props.getProperty("exception.stacktrace");
                    if (exMsg != null) {
                        LOG.warn("Launcher exception: {0}{E}{1}", exMsg, exStackTrace);
                    }
                } else {
                    errorReason = XLog.format("Launcher AM died, check Hadoop LOG for job [{0}:{1}]", action.getTrackerUri(), action.getExternalId());
                    LOG.warn(errorReason);
                }
                context.setExecutionData(FAILED_KILLED, null);
            }
        } else {
            context.setExternalStatus(YarnApplicationState.RUNNING.toString());
            LOG.info(XLog.STD, "checking action, hadoop job ID [{0}] status [RUNNING]", action.getExternalId());
        }
    } catch (Exception ex) {
        LOG.warn("Exception in check(). Message[{0}]", ex.getMessage(), ex);
        throw convertException(ex);
    } finally {
        if (yarnClient != null) {
            IOUtils.closeQuietly(yarnClient);
        }
    }
}