Examples with WorkflowApp org.apache.oozie.workflow.WorkflowApp used on opensource projects

Example 6 with WorkflowApp

use of org.apache.oozie.workflow.WorkflowApp in project oozie by apache.

the class TestDBWorkflowStore method _testInsertWF.

private void _testInsertWF() throws Exception {
    store.beginTrx();
    WorkflowApp app = new LiteWorkflowApp("testApp", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    Configuration conf1 = new Configuration();
    conf1.set(OozieClient.APP_PATH, "testPath");
    conf1.set(OozieClient.LOG_TOKEN, "testToken");
    conf1.set(OozieClient.USER_NAME, getTestUser());
    wfBean1 = createWorkflow(app, conf1, "auth");
    Configuration conf2 = new Configuration();
    conf2.set(OozieClient.APP_PATH, "testPath");
    conf2.set(OozieClient.LOG_TOKEN, "testToken");
    conf2.set(OozieClient.USER_NAME, getTestUser2());
    wfBean2 = createWorkflow(app, conf2, "auth");
    store.insertWorkflow(wfBean1);
    store.insertWorkflow(wfBean2);
    store.commitTrx();
/*
 * SqlStatement s = getCount(OozieTable.WORKFLOWS); ResultSet rs =
 * s.prepareAndSetValues(conn).executeQuery(); rs.next(); assertEquals(2,
 * rs.getInt(1)); rs.close();
 *
 * s = getCount(OozieTable.WORKFLOWS).where(isEqual(WF_id, wfBean1.getId())); rs =
 * s.prepareAndSetValues(conn).executeQuery(); rs.next(); assertEquals(1,
 * rs.getInt(1)); rs.close();
 *
 * s = getCount(OozieTable.WORKFLOWS).where(isEqual(WF_id, wfBean2.getId())); rs =
 * s.prepareAndSetValues(conn).executeQuery(); rs.next(); assertEquals(1,
 * rs.getInt(1)); rs.close();
 */
}

Example 7 with WorkflowApp

use of org.apache.oozie.workflow.WorkflowApp in project oozie by apache.

the class ReRunXCommand method setupReRun.

private void setupReRun() throws CommandException {
    InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
    LogUtils.setLogInfo(wfBean);
    WorkflowInstance oldWfInstance = this.wfBean.getWorkflowInstance();
    WorkflowInstance newWfInstance;
    String appPath = null;
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    try {
        XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
        WorkflowApp app = wps.parseDef(conf, null);
        XConfiguration protoActionConf = wps.createProtoActionConf(conf, true);
        WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
        appPath = conf.get(OozieClient.APP_PATH);
        URI uri = new URI(appPath);
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        Configuration fsConf = has.createConfiguration(uri.getAuthority());
        FileSystem fs = has.createFileSystem(wfBean.getUser(), uri, fsConf);
        Path configDefault = null;
        // app path could be a directory
        Path path = new Path(uri.getPath());
        if (!fs.isFile(path)) {
            configDefault = new Path(path, SubmitXCommand.CONFIG_DEFAULT);
        } else {
            configDefault = new Path(path.getParent(), SubmitXCommand.CONFIG_DEFAULT);
        }
        if (fs.exists(configDefault)) {
            Configuration defaultConf = new XConfiguration(fs.open(configDefault));
            PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
            XConfiguration.injectDefaults(defaultConf, conf);
        }
        PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
        // Resolving all variables in the job properties. This ensures the Hadoop Configuration semantics are
        // preserved. The Configuration.get function within XConfiguration.resolve() works recursively to get the
        // final value corresponding to a key in the map Resetting the conf to contain all the resolved values is
        // necessary to ensure propagation of Oozie properties to Hadoop calls downstream
        conf = ((XConfiguration) conf).resolve();
        try {
            newWfInstance = workflowLib.createInstance(app, conf, jobId);
        } catch (WorkflowException e) {
            throw new CommandException(e);
        }
        String appName = ELUtils.resolveAppName(app.getName(), conf);
        if (SLAService.isEnabled()) {
            Element wfElem = XmlUtils.parseXml(app.getDefinition());
            ELEvaluator evalSla = SubmitXCommand.createELEvaluatorForGroup(conf, "wf-sla-submit");
            Element eSla = XmlUtils.getSLAElement(wfElem);
            String jobSlaXml = null;
            if (eSla != null) {
                jobSlaXml = SubmitXCommand.resolveSla(eSla, evalSla);
            }
            writeSLARegistration(wfElem, jobSlaXml, newWfInstance.getId(), conf.get(SubWorkflowActionExecutor.PARENT_ID), conf.get(OozieClient.USER_NAME), appName, evalSla);
        }
        wfBean.setAppName(appName);
        wfBean.setProtoActionConf(protoActionConf.toXmlString());
    } catch (WorkflowException ex) {
        throw new CommandException(ex);
    } catch (IOException ex) {
        throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
    } catch (HadoopAccessorException ex) {
        throw new CommandException(ex);
    } catch (URISyntaxException ex) {
        throw new CommandException(ErrorCode.E0711, appPath, ex.getMessage(), ex);
    } catch (Exception ex) {
        throw new CommandException(ErrorCode.E1007, ex.getMessage(), ex);
    }
    for (int i = 0; i < actions.size(); i++) {
        // action will be used to rerun the job.
        if (!nodesToSkip.contains(actions.get(i).getName()) && !(conf.getBoolean(OozieClient.RERUN_FAIL_NODES, false) && SubWorkflowActionExecutor.ACTION_TYPE.equals(actions.get(i).getType()))) {
            deleteList.add(actions.get(i));
            LOG.info("Deleting Action[{0}] for re-run", actions.get(i).getId());
        } else {
            copyActionData(newWfInstance, oldWfInstance);
        }
    }
    wfBean.setAppPath(conf.get(OozieClient.APP_PATH));
    wfBean.setConf(XmlUtils.prettyPrint(conf).toString());
    wfBean.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
    wfBean.setUser(conf.get(OozieClient.USER_NAME));
    String group = ConfigUtils.getWithDeprecatedCheck(conf, OozieClient.JOB_ACL, OozieClient.GROUP_NAME, null);
    wfBean.setGroup(group);
    wfBean.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
    wfBean.setEndTime(null);
    wfBean.setRun(wfBean.getRun() + 1);
    wfBean.setStatus(WorkflowJob.Status.PREP);
    wfBean.setWorkflowInstance(newWfInstance);
    try {
        wfBean.setLastModifiedTime(new Date());
        updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_RERUN, wfBean));
        // call JPAExecutor to do the bulk writes
        BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(null, updateList, deleteList);
    } catch (JPAExecutorException je) {
        throw new CommandException(je);
    } finally {
        updateParentIfNecessary(wfBean);
    }
}

Example 8 with WorkflowApp

use of org.apache.oozie.workflow.WorkflowApp in project oozie by apache.

the class TestEventGeneration method testWorkflowJobEvent.

@Test
public void testWorkflowJobEvent() throws Exception {
    assertEquals(0, queue.size());
    WorkflowApp app = new LiteWorkflowApp("testApp", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "fs-node")).addNode(new ActionNodeDef("fs-node", "", TestLiteWorkflowLib.TestActionNodeHandler.class, "end", "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    WorkflowJobBean job = addRecordToWfJobTable(app, WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
    // Starting job
    new StartXCommand(job.getId()).call();
    WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(job.getId());
    job = jpaService.execute(wfJobGetCmd);
    assertEquals(WorkflowJob.Status.RUNNING, job.getStatus());
    assertEquals(1, queue.size());
    JobEvent event = (JobEvent) queue.poll();
    assertNotNull(event);
    assertEquals(EventStatus.STARTED, event.getEventStatus());
    assertEquals(AppType.WORKFLOW_JOB, event.getAppType());
    assertEquals(job.getId(), event.getId());
    assertEquals(job.getUser(), event.getUser());
    assertEquals(job.getAppName(), event.getAppName());
    assertEquals(job.getStartTime(), event.getStartTime());
    assertEquals(0, queue.size());
    // Suspending job
    new SuspendXCommand(job.getId()).call();
    job = jpaService.execute(wfJobGetCmd);
    assertEquals(WorkflowJob.Status.SUSPENDED, job.getStatus());
    assertEquals(1, queue.size());
    event = (JobEvent) queue.poll();
    assertNotNull(event);
    assertEquals(EventStatus.SUSPEND, event.getEventStatus());
    assertEquals(AppType.WORKFLOW_JOB, event.getAppType());
    assertEquals(job.getId(), event.getId());
    assertEquals(job.getUser(), event.getUser());
    assertEquals(job.getAppName(), event.getAppName());
    assertEquals(0, queue.size());
    // Resuming job
    new ResumeXCommand(job.getId()).call();
    job = jpaService.execute(wfJobGetCmd);
    assertEquals(WorkflowJob.Status.RUNNING, job.getStatus());
    assertEquals(1, queue.size());
    event = (JobEvent) queue.poll();
    assertNotNull(event);
    assertEquals(AppType.WORKFLOW_JOB, event.getAppType());
    assertEquals(job.getId(), event.getId());
    assertEquals(job.getUser(), event.getUser());
    assertEquals(job.getAppName(), event.getAppName());
    assertEquals(job.getStartTime(), event.getStartTime());
    assertEquals(0, queue.size());
    // Killing job
    new KillXCommand(job.getId()).call();
    job = jpaService.execute(wfJobGetCmd);
    assertEquals(WorkflowJob.Status.KILLED, job.getStatus());
    assertEquals(1, queue.size());
    event = (JobEvent) queue.poll();
    assertNotNull(event);
    assertEquals(EventStatus.FAILURE, event.getEventStatus());
    assertEquals(AppType.WORKFLOW_JOB, event.getAppType());
    assertEquals(job.getId(), event.getId());
    assertEquals(job.getUser(), event.getUser());
    assertEquals(job.getAppName(), event.getAppName());
    assertEquals(job.getStartTime(), event.getStartTime());
    assertEquals(job.getEndTime(), event.getEndTime());
    assertEquals(0, queue.size());
    // Successful job (testing SignalX)
    job = _createWorkflowJob();
    LiteWorkflowInstance wfInstance = (LiteWorkflowInstance) job.getWorkflowInstance();
    wfInstance.start();
    job.setWorkflowInstance(wfInstance);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MODIFIED, job);
    WorkflowActionBean wfAction = jpaService.execute(new WorkflowActionGetJPAExecutor(job.getId() + "@one"));
    new SignalXCommand(job.getId(), wfAction.getId()).call();
    job = jpaService.execute(new WorkflowJobGetJPAExecutor(job.getId()));
    assertEquals(WorkflowJob.Status.SUCCEEDED, job.getStatus());
    assertEquals(1, queue.size());
    event = (JobEvent) queue.poll();
    assertNotNull(event);
    assertEquals(AppType.WORKFLOW_JOB, event.getAppType());
    assertEquals(job.getId(), event.getId());
    assertEquals(job.getUser(), event.getUser());
    assertEquals(job.getAppName(), event.getAppName());
    assertEquals(job.getStartTime(), event.getStartTime());
    assertEquals(job.getEndTime(), event.getEndTime());
}

Example 9 with WorkflowApp

use of org.apache.oozie.workflow.WorkflowApp in project oozie by apache.

the class SubmitXCommand method execute.

@Override
protected String execute() throws CommandException {
    InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    try {
        XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
        String user = conf.get(OozieClient.USER_NAME);
        URI uri = new URI(conf.get(OozieClient.APP_PATH));
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        Configuration fsConf = has.createConfiguration(uri.getAuthority());
        FileSystem fs = has.createFileSystem(user, uri, fsConf);
        Path configDefault = null;
        Configuration defaultConf = null;
        // app path could be a directory
        Path path = new Path(uri.getPath());
        if (!fs.isFile(path)) {
            configDefault = new Path(path, CONFIG_DEFAULT);
        } else {
            configDefault = new Path(path.getParent(), CONFIG_DEFAULT);
        }
        if (fs.exists(configDefault)) {
            try {
                defaultConf = new XConfiguration(fs.open(configDefault));
                PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
                XConfiguration.injectDefaults(defaultConf, conf);
            } catch (IOException ex) {
                throw new IOException("default configuration file, " + ex.getMessage(), ex);
            }
        }
        if (defaultConf != null) {
            defaultConf = resolveDefaultConfVariables(defaultConf);
        }
        WorkflowApp app = wps.parseDef(conf, defaultConf);
        XConfiguration protoActionConf = wps.createProtoActionConf(conf, true);
        WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
        PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
        // Resolving all variables in the job properties.
        // This ensures the Hadoop Configuration semantics is preserved.
        XConfiguration resolvedVarsConf = new XConfiguration();
        for (Map.Entry<String, String> entry : conf) {
            resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
        }
        conf = resolvedVarsConf;
        WorkflowInstance wfInstance;
        try {
            wfInstance = workflowLib.createInstance(app, conf);
        } catch (WorkflowException e) {
            throw new StoreException(e);
        }
        Configuration conf = wfInstance.getConf();
        // System.out.println("WF INSTANCE CONF:");
        // System.out.println(XmlUtils.prettyPrint(conf).toString());
        WorkflowJobBean workflow = new WorkflowJobBean();
        workflow.setId(wfInstance.getId());
        workflow.setAppName(ELUtils.resolveAppName(app.getName(), conf));
        workflow.setAppPath(conf.get(OozieClient.APP_PATH));
        workflow.setConf(XmlUtils.prettyPrint(conf).toString());
        workflow.setProtoActionConf(protoActionConf.toXmlString());
        workflow.setCreatedTime(new Date());
        workflow.setLastModifiedTime(new Date());
        workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
        workflow.setStatus(WorkflowJob.Status.PREP);
        workflow.setRun(0);
        workflow.setUser(conf.get(OozieClient.USER_NAME));
        workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
        workflow.setWorkflowInstance(wfInstance);
        workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
        // Set parent id if it doesn't already have one (for subworkflows)
        if (workflow.getParentId() == null) {
            workflow.setParentId(conf.get(SubWorkflowActionExecutor.PARENT_ID));
        }
        // Set to coord action Id if workflow submitted through coordinator
        if (workflow.getParentId() == null) {
            workflow.setParentId(parentId);
        }
        LogUtils.setLogInfo(workflow);
        LOG.debug("Workflow record created, Status [{0}]", workflow.getStatus());
        Element wfElem = XmlUtils.parseXml(app.getDefinition());
        ELEvaluator evalSla = createELEvaluatorForGroup(conf, "wf-sla-submit");
        String jobSlaXml = verifySlaElements(wfElem, evalSla);
        if (!dryrun) {
            writeSLARegistration(wfElem, jobSlaXml, workflow.getId(), workflow.getParentId(), workflow.getUser(), workflow.getGroup(), workflow.getAppName(), LOG, evalSla);
            workflow.setSlaXml(jobSlaXml);
            // System.out.println("SlaXml :"+ slaXml);
            // store.insertWorkflow(workflow);
            insertList.add(workflow);
            JPAService jpaService = Services.get().get(JPAService.class);
            if (jpaService != null) {
                try {
                    BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, null, null);
                } catch (JPAExecutorException je) {
                    throw new CommandException(je);
                }
            } else {
                LOG.error(ErrorCode.E0610);
                return null;
            }
            return workflow.getId();
        } else {
            // Checking variable substitution for dryrun
            ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, null, false, false);
            Element workflowXml = XmlUtils.parseXml(app.getDefinition());
            removeSlaElements(workflowXml);
            String workflowXmlString = XmlUtils.removeComments(XmlUtils.prettyPrint(workflowXml).toString());
            workflowXmlString = context.getELEvaluator().evaluate(workflowXmlString, String.class);
            workflowXml = XmlUtils.parseXml(workflowXmlString);
            Iterator<Element> it = workflowXml.getDescendants(new ElementFilter("job-xml"));
            // Checking all variable substitutions in job-xml files
            while (it.hasNext()) {
                Element e = it.next();
                String jobXml = e.getTextTrim();
                Path xmlPath = new Path(workflow.getAppPath(), jobXml);
                Configuration jobXmlConf = new XConfiguration(fs.open(xmlPath));
                String jobXmlConfString = XmlUtils.prettyPrint(jobXmlConf).toString();
                jobXmlConfString = XmlUtils.removeComments(jobXmlConfString);
                context.getELEvaluator().evaluate(jobXmlConfString, String.class);
            }
            return "OK";
        }
    } catch (WorkflowException ex) {
        throw new CommandException(ex);
    } catch (HadoopAccessorException ex) {
        throw new CommandException(ex);
    } catch (Exception ex) {
        throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
    }
}

Example 10 with WorkflowApp

use of org.apache.oozie.workflow.WorkflowApp in project oozie by apache.

the class TestJavaActionExecutor method addRecordToWfJobTable.

private WorkflowJobBean addRecordToWfJobTable(String wfId, String wfxml, Map<String, String> otherProps) throws Exception {
    WorkflowApp app = new LiteWorkflowApp("testApp", wfxml, new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "start")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    Configuration conf = getHadoopAccessorService().createConfiguration(new URI(getNameNodeUri()).getAuthority());
    conf.set(OozieClient.APP_PATH, getNameNodeUri() + "/testPath");
    conf.set(OozieClient.LOG_TOKEN, "testToken");
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set("property3", "prop3");
    conf.set("value3", "val3");
    if (otherProps != null) {
        for (Map.Entry<String, String> ent : otherProps.entrySet()) {
            conf.set(ent.getKey(), ent.getValue());
        }
    }
    WorkflowJobBean wfBean = createWorkflow(app, conf, "auth");
    wfBean.setId(wfId);
    wfBean.setStatus(WorkflowJob.Status.SUCCEEDED);
    WorkflowActionBean action = new WorkflowActionBean();
    action.setName("test");
    action.setId(Services.get().get(UUIDService.class).generateChildId(wfBean.getId(), "test"));
    wfBean.getActions().add(action);
    return wfBean;
}