Examples with WorkflowAppService org.apache.oozie.service.WorkflowAppService used on opensource projects

Example 1 with WorkflowAppService

use of org.apache.oozie.service.WorkflowAppService in project oozie by apache.

the class TestPurgeXCommand method createWorkflow.

@Override
protected WorkflowJobBean createWorkflow(WorkflowApp app, Configuration conf, WorkflowJob.Status jobStatus, WorkflowInstance.Status instanceStatus) throws Exception {
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    Configuration protoActionConf = wps.createProtoActionConf(conf, true);
    WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
    WorkflowInstance wfInstance = workflowLib.createInstance(app, conf);
    ((LiteWorkflowInstance) wfInstance).setStatus(instanceStatus);
    WorkflowJobBean workflow = new WorkflowJobBean();
    workflow.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.WORKFLOW));
    workflow.setAppName(app.getName());
    workflow.setAppPath(conf.get(OozieClient.APP_PATH));
    workflow.setConf(XmlUtils.prettyPrint(conf).toString());
    workflow.setProtoActionConf(XmlUtils.prettyPrint(protoActionConf).toString());
    workflow.setCreatedTime(new Date());
    workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
    workflow.setStatus(jobStatus);
    workflow.setRun(0);
    workflow.setUser(conf.get(OozieClient.USER_NAME));
    workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
    workflow.setWorkflowInstance(wfInstance);
    workflow.setStartTime(DateUtils.parseDateOozieTZ("2009-12-18T01:00Z"));
    workflow.setEndTime(DateUtils.parseDateOozieTZ("2009-12-18T03:00Z"));
    return workflow;
}

Example 2 with WorkflowAppService

use of org.apache.oozie.service.WorkflowAppService in project oozie by apache.

the class XDataTestCase method createWorkflow.

/**
 * Create workflow job bean
 *
 * @param app workflow app
 * @param conf workflow configuration
 * @param authToken auth token
 * @param jobStatus workflow job status
 * @param instanceStatus workflow instance status
 * @return workflow job bean
 * @throws Exception thrown if unable to create workflow job bean
 */
protected WorkflowJobBean createWorkflow(WorkflowApp app, Configuration conf, WorkflowJob.Status jobStatus, WorkflowInstance.Status instanceStatus) throws Exception {
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    Configuration protoActionConf = wps.createProtoActionConf(conf, true);
    WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
    WorkflowInstance wfInstance = workflowLib.createInstance(app, conf);
    ((LiteWorkflowInstance) wfInstance).setStatus(instanceStatus);
    WorkflowJobBean workflow = new WorkflowJobBean();
    workflow.setId(wfInstance.getId());
    workflow.setExternalId("extid");
    workflow.setAppName(app.getName());
    workflow.setAppPath(conf.get(OozieClient.APP_PATH));
    workflow.setConf(XmlUtils.prettyPrint(conf).toString());
    workflow.setProtoActionConf(XmlUtils.prettyPrint(protoActionConf).toString());
    workflow.setCreatedTime(new Date());
    workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
    workflow.setStatus(jobStatus);
    workflow.setRun(0);
    workflow.setUser(conf.get(OozieClient.USER_NAME));
    workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
    workflow.setWorkflowInstance(wfInstance);
    workflow.setSlaXml("<sla></sla>");
    return workflow;
}

Example 3 with WorkflowAppService

use of org.apache.oozie.service.WorkflowAppService in project oozie by apache.

the class ReRunXCommand method setupReRun.

private void setupReRun() throws CommandException {
    InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
    LogUtils.setLogInfo(wfBean);
    WorkflowInstance oldWfInstance = this.wfBean.getWorkflowInstance();
    WorkflowInstance newWfInstance;
    String appPath = null;
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    try {
        XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
        WorkflowApp app = wps.parseDef(conf, null);
        XConfiguration protoActionConf = wps.createProtoActionConf(conf, true);
        WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
        appPath = conf.get(OozieClient.APP_PATH);
        URI uri = new URI(appPath);
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        Configuration fsConf = has.createConfiguration(uri.getAuthority());
        FileSystem fs = has.createFileSystem(wfBean.getUser(), uri, fsConf);
        Path configDefault = null;
        // app path could be a directory
        Path path = new Path(uri.getPath());
        if (!fs.isFile(path)) {
            configDefault = new Path(path, SubmitXCommand.CONFIG_DEFAULT);
        } else {
            configDefault = new Path(path.getParent(), SubmitXCommand.CONFIG_DEFAULT);
        }
        if (fs.exists(configDefault)) {
            Configuration defaultConf = new XConfiguration(fs.open(configDefault));
            PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
            XConfiguration.injectDefaults(defaultConf, conf);
        }
        PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
        // Resolving all variables in the job properties. This ensures the Hadoop Configuration semantics are
        // preserved. The Configuration.get function within XConfiguration.resolve() works recursively to get the
        // final value corresponding to a key in the map Resetting the conf to contain all the resolved values is
        // necessary to ensure propagation of Oozie properties to Hadoop calls downstream
        conf = ((XConfiguration) conf).resolve();
        try {
            newWfInstance = workflowLib.createInstance(app, conf, jobId);
        } catch (WorkflowException e) {
            throw new CommandException(e);
        }
        String appName = ELUtils.resolveAppName(app.getName(), conf);
        if (SLAService.isEnabled()) {
            Element wfElem = XmlUtils.parseXml(app.getDefinition());
            ELEvaluator evalSla = SubmitXCommand.createELEvaluatorForGroup(conf, "wf-sla-submit");
            Element eSla = XmlUtils.getSLAElement(wfElem);
            String jobSlaXml = null;
            if (eSla != null) {
                jobSlaXml = SubmitXCommand.resolveSla(eSla, evalSla);
            }
            writeSLARegistration(wfElem, jobSlaXml, newWfInstance.getId(), conf.get(SubWorkflowActionExecutor.PARENT_ID), conf.get(OozieClient.USER_NAME), appName, evalSla);
        }
        wfBean.setAppName(appName);
        wfBean.setProtoActionConf(protoActionConf.toXmlString());
    } catch (WorkflowException ex) {
        throw new CommandException(ex);
    } catch (IOException ex) {
        throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
    } catch (HadoopAccessorException ex) {
        throw new CommandException(ex);
    } catch (URISyntaxException ex) {
        throw new CommandException(ErrorCode.E0711, appPath, ex.getMessage(), ex);
    } catch (Exception ex) {
        throw new CommandException(ErrorCode.E1007, ex.getMessage(), ex);
    }
    for (int i = 0; i < actions.size(); i++) {
        // action will be used to rerun the job.
        if (!nodesToSkip.contains(actions.get(i).getName()) && !(conf.getBoolean(OozieClient.RERUN_FAIL_NODES, false) && SubWorkflowActionExecutor.ACTION_TYPE.equals(actions.get(i).getType()))) {
            deleteList.add(actions.get(i));
            LOG.info("Deleting Action[{0}] for re-run", actions.get(i).getId());
        } else {
            copyActionData(newWfInstance, oldWfInstance);
        }
    }
    wfBean.setAppPath(conf.get(OozieClient.APP_PATH));
    wfBean.setConf(XmlUtils.prettyPrint(conf).toString());
    wfBean.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
    wfBean.setUser(conf.get(OozieClient.USER_NAME));
    String group = ConfigUtils.getWithDeprecatedCheck(conf, OozieClient.JOB_ACL, OozieClient.GROUP_NAME, null);
    wfBean.setGroup(group);
    wfBean.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
    wfBean.setEndTime(null);
    wfBean.setRun(wfBean.getRun() + 1);
    wfBean.setStatus(WorkflowJob.Status.PREP);
    wfBean.setWorkflowInstance(newWfInstance);
    try {
        wfBean.setLastModifiedTime(new Date());
        updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_RERUN, wfBean));
        // call JPAExecutor to do the bulk writes
        BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(null, updateList, deleteList);
    } catch (JPAExecutorException je) {
        throw new CommandException(je);
    } finally {
        updateParentIfNecessary(wfBean);
    }
}

Example 4 with WorkflowAppService

use of org.apache.oozie.service.WorkflowAppService in project oozie by apache.

the class TestSubmitMRXCommand method testWFXmlGeneration.

public void testWFXmlGeneration() throws Exception {
    Configuration conf = new Configuration(false);
    conf.set(XOozieClient.RM, "jobtracker");
    conf.set(XOozieClient.NN, "namenode");
    conf.set(OozieClient.LIBPATH, "libpath");
    conf.set("mapred.mapper.class", "A.Mapper");
    conf.set("mapred.reducer.class", "A.Reducer");
    conf.set(XOozieClient.FILES, "/user/oozie/input1.txt,/user/oozie/input2.txt#my.txt");
    conf.set(XOozieClient.ARCHIVES, "/user/oozie/udf1.jar,/user/oozie/udf2.jar#my.jar");
    SubmitMRXCommand submitMRCmd = new SubmitMRXCommand(conf);
    String xml = submitMRCmd.getWorkflowXml(conf);
    XLog.getLog(getClass()).info("xml = " + xml);
    // verifying is a valid WF
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    wps.parseDef(xml, conf);
    Element wfE = XmlUtils.parseXml(xml);
    Namespace ns = wfE.getNamespace();
    Element actionE = wfE.getChild("action", ns).getChild("map-reduce", ns);
    Element confE = actionE.getChild("configuration", ns);
    Map<String, String> props = new HashMap<String, String>();
    for (Object prop : confE.getChildren("property", ns)) {
        Element propE = (Element) prop;
        String name = propE.getChild("name", ns).getTextTrim();
        String value = propE.getChild("value", ns).getTextTrim();
        props.put(name, value);
    }
    Map<String, String> expected = new HashMap<String, String>();
    expected.put("mapred.mapper.class", "A.Mapper");
    expected.put("mapred.reducer.class", "A.Reducer");
    for (Map.Entry<String, String> entry : expected.entrySet()) {
        assertEquals(entry.getValue(), expected.get(entry.getKey()));
    }
    assertEquals("/user/oozie/input1.txt#input1.txt", ((Element) actionE.getChildren("file", ns).get(0)).getTextTrim());
    assertEquals("/user/oozie/input2.txt#my.txt", ((Element) actionE.getChildren("file", ns).get(1)).getTextTrim());
    assertEquals("/user/oozie/udf1.jar#udf1.jar", ((Element) actionE.getChildren("archive", ns).get(0)).getTextTrim());
    assertEquals("/user/oozie/udf2.jar#my.jar", ((Element) actionE.getChildren("archive", ns).get(1)).getTextTrim());
}

Example 5 with WorkflowAppService

use of org.apache.oozie.service.WorkflowAppService in project oozie by apache.

the class SubmitXCommand method execute.

@Override
protected String execute() throws CommandException {
    InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    try {
        XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
        String user = conf.get(OozieClient.USER_NAME);
        URI uri = new URI(conf.get(OozieClient.APP_PATH));
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        Configuration fsConf = has.createConfiguration(uri.getAuthority());
        FileSystem fs = has.createFileSystem(user, uri, fsConf);
        Path configDefault = null;
        Configuration defaultConf = null;
        // app path could be a directory
        Path path = new Path(uri.getPath());
        if (!fs.isFile(path)) {
            configDefault = new Path(path, CONFIG_DEFAULT);
        } else {
            configDefault = new Path(path.getParent(), CONFIG_DEFAULT);
        }
        if (fs.exists(configDefault)) {
            try {
                defaultConf = new XConfiguration(fs.open(configDefault));
                PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
                XConfiguration.injectDefaults(defaultConf, conf);
            } catch (IOException ex) {
                throw new IOException("default configuration file, " + ex.getMessage(), ex);
            }
        }
        if (defaultConf != null) {
            defaultConf = resolveDefaultConfVariables(defaultConf);
        }
        WorkflowApp app = wps.parseDef(conf, defaultConf);
        XConfiguration protoActionConf = wps.createProtoActionConf(conf, true);
        WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
        PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
        // Resolving all variables in the job properties.
        // This ensures the Hadoop Configuration semantics is preserved.
        XConfiguration resolvedVarsConf = new XConfiguration();
        for (Map.Entry<String, String> entry : conf) {
            resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
        }
        conf = resolvedVarsConf;
        WorkflowInstance wfInstance;
        try {
            wfInstance = workflowLib.createInstance(app, conf);
        } catch (WorkflowException e) {
            throw new StoreException(e);
        }
        Configuration conf = wfInstance.getConf();
        // System.out.println("WF INSTANCE CONF:");
        // System.out.println(XmlUtils.prettyPrint(conf).toString());
        WorkflowJobBean workflow = new WorkflowJobBean();
        workflow.setId(wfInstance.getId());
        workflow.setAppName(ELUtils.resolveAppName(app.getName(), conf));
        workflow.setAppPath(conf.get(OozieClient.APP_PATH));
        workflow.setConf(XmlUtils.prettyPrint(conf).toString());
        workflow.setProtoActionConf(protoActionConf.toXmlString());
        workflow.setCreatedTime(new Date());
        workflow.setLastModifiedTime(new Date());
        workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
        workflow.setStatus(WorkflowJob.Status.PREP);
        workflow.setRun(0);
        workflow.setUser(conf.get(OozieClient.USER_NAME));
        workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
        workflow.setWorkflowInstance(wfInstance);
        workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
        // Set parent id if it doesn't already have one (for subworkflows)
        if (workflow.getParentId() == null) {
            workflow.setParentId(conf.get(SubWorkflowActionExecutor.PARENT_ID));
        }
        // Set to coord action Id if workflow submitted through coordinator
        if (workflow.getParentId() == null) {
            workflow.setParentId(parentId);
        }
        LogUtils.setLogInfo(workflow);
        LOG.debug("Workflow record created, Status [{0}]", workflow.getStatus());
        Element wfElem = XmlUtils.parseXml(app.getDefinition());
        ELEvaluator evalSla = createELEvaluatorForGroup(conf, "wf-sla-submit");
        String jobSlaXml = verifySlaElements(wfElem, evalSla);
        if (!dryrun) {
            writeSLARegistration(wfElem, jobSlaXml, workflow.getId(), workflow.getParentId(), workflow.getUser(), workflow.getGroup(), workflow.getAppName(), LOG, evalSla);
            workflow.setSlaXml(jobSlaXml);
            // System.out.println("SlaXml :"+ slaXml);
            // store.insertWorkflow(workflow);
            insertList.add(workflow);
            JPAService jpaService = Services.get().get(JPAService.class);
            if (jpaService != null) {
                try {
                    BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, null, null);
                } catch (JPAExecutorException je) {
                    throw new CommandException(je);
                }
            } else {
                LOG.error(ErrorCode.E0610);
                return null;
            }
            return workflow.getId();
        } else {
            // Checking variable substitution for dryrun
            ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, null, false, false);
            Element workflowXml = XmlUtils.parseXml(app.getDefinition());
            removeSlaElements(workflowXml);
            String workflowXmlString = XmlUtils.removeComments(XmlUtils.prettyPrint(workflowXml).toString());
            workflowXmlString = context.getELEvaluator().evaluate(workflowXmlString, String.class);
            workflowXml = XmlUtils.parseXml(workflowXmlString);
            Iterator<Element> it = workflowXml.getDescendants(new ElementFilter("job-xml"));
            // Checking all variable substitutions in job-xml files
            while (it.hasNext()) {
                Element e = it.next();
                String jobXml = e.getTextTrim();
                Path xmlPath = new Path(workflow.getAppPath(), jobXml);
                Configuration jobXmlConf = new XConfiguration(fs.open(xmlPath));
                String jobXmlConfString = XmlUtils.prettyPrint(jobXmlConf).toString();
                jobXmlConfString = XmlUtils.removeComments(jobXmlConfString);
                context.getELEvaluator().evaluate(jobXmlConfString, String.class);
            }
            return "OK";
        }
    } catch (WorkflowException ex) {
        throw new CommandException(ex);
    } catch (HadoopAccessorException ex) {
        throw new CommandException(ex);
    } catch (Exception ex) {
        throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
    }
}