Examples with ELEvaluator org.apache.oozie.util.ELEvaluator used on opensource projects

Example 6 with ELEvaluator

use of org.apache.oozie.util.ELEvaluator in project oozie by apache.

the class ReRunXCommand method setupReRun.

private void setupReRun() throws CommandException {
    InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
    LogUtils.setLogInfo(wfBean);
    WorkflowInstance oldWfInstance = this.wfBean.getWorkflowInstance();
    WorkflowInstance newWfInstance;
    String appPath = null;
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    try {
        XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
        WorkflowApp app = wps.parseDef(conf, null);
        XConfiguration protoActionConf = wps.createProtoActionConf(conf, true);
        WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
        appPath = conf.get(OozieClient.APP_PATH);
        URI uri = new URI(appPath);
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        Configuration fsConf = has.createConfiguration(uri.getAuthority());
        FileSystem fs = has.createFileSystem(wfBean.getUser(), uri, fsConf);
        Path configDefault = null;
        // app path could be a directory
        Path path = new Path(uri.getPath());
        if (!fs.isFile(path)) {
            configDefault = new Path(path, SubmitXCommand.CONFIG_DEFAULT);
        } else {
            configDefault = new Path(path.getParent(), SubmitXCommand.CONFIG_DEFAULT);
        }
        if (fs.exists(configDefault)) {
            Configuration defaultConf = new XConfiguration(fs.open(configDefault));
            PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
            XConfiguration.injectDefaults(defaultConf, conf);
        }
        PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
        // Resolving all variables in the job properties. This ensures the Hadoop Configuration semantics are
        // preserved. The Configuration.get function within XConfiguration.resolve() works recursively to get the
        // final value corresponding to a key in the map Resetting the conf to contain all the resolved values is
        // necessary to ensure propagation of Oozie properties to Hadoop calls downstream
        conf = ((XConfiguration) conf).resolve();
        try {
            newWfInstance = workflowLib.createInstance(app, conf, jobId);
        } catch (WorkflowException e) {
            throw new CommandException(e);
        }
        String appName = ELUtils.resolveAppName(app.getName(), conf);
        if (SLAService.isEnabled()) {
            Element wfElem = XmlUtils.parseXml(app.getDefinition());
            ELEvaluator evalSla = SubmitXCommand.createELEvaluatorForGroup(conf, "wf-sla-submit");
            Element eSla = XmlUtils.getSLAElement(wfElem);
            String jobSlaXml = null;
            if (eSla != null) {
                jobSlaXml = SubmitXCommand.resolveSla(eSla, evalSla);
            }
            writeSLARegistration(wfElem, jobSlaXml, newWfInstance.getId(), conf.get(SubWorkflowActionExecutor.PARENT_ID), conf.get(OozieClient.USER_NAME), appName, evalSla);
        }
        wfBean.setAppName(appName);
        wfBean.setProtoActionConf(protoActionConf.toXmlString());
    } catch (WorkflowException ex) {
        throw new CommandException(ex);
    } catch (IOException ex) {
        throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
    } catch (HadoopAccessorException ex) {
        throw new CommandException(ex);
    } catch (URISyntaxException ex) {
        throw new CommandException(ErrorCode.E0711, appPath, ex.getMessage(), ex);
    } catch (Exception ex) {
        throw new CommandException(ErrorCode.E1007, ex.getMessage(), ex);
    }
    for (int i = 0; i < actions.size(); i++) {
        // action will be used to rerun the job.
        if (!nodesToSkip.contains(actions.get(i).getName()) && !(conf.getBoolean(OozieClient.RERUN_FAIL_NODES, false) && SubWorkflowActionExecutor.ACTION_TYPE.equals(actions.get(i).getType()))) {
            deleteList.add(actions.get(i));
            LOG.info("Deleting Action[{0}] for re-run", actions.get(i).getId());
        } else {
            copyActionData(newWfInstance, oldWfInstance);
        }
    }
    wfBean.setAppPath(conf.get(OozieClient.APP_PATH));
    wfBean.setConf(XmlUtils.prettyPrint(conf).toString());
    wfBean.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
    wfBean.setUser(conf.get(OozieClient.USER_NAME));
    String group = ConfigUtils.getWithDeprecatedCheck(conf, OozieClient.JOB_ACL, OozieClient.GROUP_NAME, null);
    wfBean.setGroup(group);
    wfBean.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
    wfBean.setEndTime(null);
    wfBean.setRun(wfBean.getRun() + 1);
    wfBean.setStatus(WorkflowJob.Status.PREP);
    wfBean.setWorkflowInstance(newWfInstance);
    try {
        wfBean.setLastModifiedTime(new Date());
        updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_RERUN, wfBean));
        // call JPAExecutor to do the bulk writes
        BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(null, updateList, deleteList);
    } catch (JPAExecutorException je) {
        throw new CommandException(je);
    } finally {
        updateParentIfNecessary(wfBean);
    }
}

Example 7 with ELEvaluator

use of org.apache.oozie.util.ELEvaluator in project oozie by apache.

the class SignalXCommand method resolveSla.

private String resolveSla(Element eSla, Configuration conf) throws CommandException {
    String slaXml = null;
    try {
        ELEvaluator evalSla = SubmitXCommand.createELEvaluatorForGroup(conf, "wf-sla-submit");
        slaXml = SubmitXCommand.resolveSla(eSla, evalSla);
    } catch (Exception e) {
        throw new CommandException(ErrorCode.E1004, e.getMessage(), e);
    }
    return slaXml;
}

Example 8 with ELEvaluator

use of org.apache.oozie.util.ELEvaluator in project oozie by apache.

the class TestCoordELEvaluator method testCreateURIELEvaluator.

public void testCreateURIELEvaluator() throws Exception {
    ELEvaluator eval = CoordELEvaluator.createURIELEvaluator("2009-08-09T23:59Z");
    String expr = "hdfs://p1/p2/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/";
    // System.out.println("OUTPUT "+ eval.evaluate(expr, String.class));
    assertEquals("hdfs://p1/p2/2009/08/09/23/59/", CoordELFunctions.evalAndWrap(eval, expr));
    expr = "hdfs://p1/p2/${YEAR}/${MONTH}/${DAY}/${MINUTE}/";
    assertEquals("hdfs://p1/p2/2009/08/09/59/", CoordELFunctions.evalAndWrap(eval, expr));
}

Example 9 with ELEvaluator

use of org.apache.oozie.util.ELEvaluator in project oozie by apache.

the class TestHCatELFunctions method testHCatTableExists.

@Test
public void testHCatTableExists() throws Exception {
    dropTable("db1", "table1", true);
    dropDatabase("db1", true);
    createDatabase("db1");
    createTable("db1", "table1");
    Configuration protoConf = new Configuration();
    protoConf.set(OozieClient.USER_NAME, getTestUser());
    protoConf.set("hadoop.job.ugi", getTestUser() + "," + "group");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, "appPath");
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set("test.dir", getTestCaseDir());
    conf.set("table1", getHCatURI("db1", "table1").toString());
    conf.set("table2", getHCatURI("db1", "table2").toString());
    LiteWorkflowApp def = new LiteWorkflowApp("name", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    LiteWorkflowInstance job = new LiteWorkflowInstance(def, conf, "wfId");
    WorkflowJobBean wf = new WorkflowJobBean();
    wf.setId(job.getId());
    wf.setAppName("name");
    wf.setAppPath("appPath");
    wf.setUser(getTestUser());
    wf.setGroup("group");
    wf.setWorkflowInstance(job);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    protoConf.writeXml(baos);
    wf.setProtoActionConf(baos.toString());
    WorkflowActionBean action = new WorkflowActionBean();
    action.setId("actionId");
    action.setName("actionName");
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, wf, action);
    assertEquals(true, (boolean) eval.evaluate("${hcat:exists(wf:conf('table1'))}", Boolean.class));
    assertEquals(false, (boolean) eval.evaluate("${hcat:exists(wf:conf('table2'))}", Boolean.class));
    dropTable("db1", "table1", true);
    dropDatabase("db1", true);
}

Example 10 with ELEvaluator

use of org.apache.oozie.util.ELEvaluator in project oozie by apache.

the class SubmitXCommand method execute.

@Override
protected String execute() throws CommandException {
    InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    try {
        XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
        String user = conf.get(OozieClient.USER_NAME);
        URI uri = new URI(conf.get(OozieClient.APP_PATH));
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        Configuration fsConf = has.createConfiguration(uri.getAuthority());
        FileSystem fs = has.createFileSystem(user, uri, fsConf);
        Path configDefault = null;
        Configuration defaultConf = null;
        // app path could be a directory
        Path path = new Path(uri.getPath());
        if (!fs.isFile(path)) {
            configDefault = new Path(path, CONFIG_DEFAULT);
        } else {
            configDefault = new Path(path.getParent(), CONFIG_DEFAULT);
        }
        if (fs.exists(configDefault)) {
            try {
                defaultConf = new XConfiguration(fs.open(configDefault));
                PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
                XConfiguration.injectDefaults(defaultConf, conf);
            } catch (IOException ex) {
                throw new IOException("default configuration file, " + ex.getMessage(), ex);
            }
        }
        if (defaultConf != null) {
            defaultConf = resolveDefaultConfVariables(defaultConf);
        }
        WorkflowApp app = wps.parseDef(conf, defaultConf);
        XConfiguration protoActionConf = wps.createProtoActionConf(conf, true);
        WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
        PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
        // Resolving all variables in the job properties.
        // This ensures the Hadoop Configuration semantics is preserved.
        XConfiguration resolvedVarsConf = new XConfiguration();
        for (Map.Entry<String, String> entry : conf) {
            resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
        }
        conf = resolvedVarsConf;
        WorkflowInstance wfInstance;
        try {
            wfInstance = workflowLib.createInstance(app, conf);
        } catch (WorkflowException e) {
            throw new StoreException(e);
        }
        Configuration conf = wfInstance.getConf();
        // System.out.println("WF INSTANCE CONF:");
        // System.out.println(XmlUtils.prettyPrint(conf).toString());
        WorkflowJobBean workflow = new WorkflowJobBean();
        workflow.setId(wfInstance.getId());
        workflow.setAppName(ELUtils.resolveAppName(app.getName(), conf));
        workflow.setAppPath(conf.get(OozieClient.APP_PATH));
        workflow.setConf(XmlUtils.prettyPrint(conf).toString());
        workflow.setProtoActionConf(protoActionConf.toXmlString());
        workflow.setCreatedTime(new Date());
        workflow.setLastModifiedTime(new Date());
        workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
        workflow.setStatus(WorkflowJob.Status.PREP);
        workflow.setRun(0);
        workflow.setUser(conf.get(OozieClient.USER_NAME));
        workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
        workflow.setWorkflowInstance(wfInstance);
        workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
        // Set parent id if it doesn't already have one (for subworkflows)
        if (workflow.getParentId() == null) {
            workflow.setParentId(conf.get(SubWorkflowActionExecutor.PARENT_ID));
        }
        // Set to coord action Id if workflow submitted through coordinator
        if (workflow.getParentId() == null) {
            workflow.setParentId(parentId);
        }
        LogUtils.setLogInfo(workflow);
        LOG.debug("Workflow record created, Status [{0}]", workflow.getStatus());
        Element wfElem = XmlUtils.parseXml(app.getDefinition());
        ELEvaluator evalSla = createELEvaluatorForGroup(conf, "wf-sla-submit");
        String jobSlaXml = verifySlaElements(wfElem, evalSla);
        if (!dryrun) {
            writeSLARegistration(wfElem, jobSlaXml, workflow.getId(), workflow.getParentId(), workflow.getUser(), workflow.getGroup(), workflow.getAppName(), LOG, evalSla);
            workflow.setSlaXml(jobSlaXml);
            // System.out.println("SlaXml :"+ slaXml);
            // store.insertWorkflow(workflow);
            insertList.add(workflow);
            JPAService jpaService = Services.get().get(JPAService.class);
            if (jpaService != null) {
                try {
                    BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, null, null);
                } catch (JPAExecutorException je) {
                    throw new CommandException(je);
                }
            } else {
                LOG.error(ErrorCode.E0610);
                return null;
            }
            return workflow.getId();
        } else {
            // Checking variable substitution for dryrun
            ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, null, false, false);
            Element workflowXml = XmlUtils.parseXml(app.getDefinition());
            removeSlaElements(workflowXml);
            String workflowXmlString = XmlUtils.removeComments(XmlUtils.prettyPrint(workflowXml).toString());
            workflowXmlString = context.getELEvaluator().evaluate(workflowXmlString, String.class);
            workflowXml = XmlUtils.parseXml(workflowXmlString);
            Iterator<Element> it = workflowXml.getDescendants(new ElementFilter("job-xml"));
            // Checking all variable substitutions in job-xml files
            while (it.hasNext()) {
                Element e = it.next();
                String jobXml = e.getTextTrim();
                Path xmlPath = new Path(workflow.getAppPath(), jobXml);
                Configuration jobXmlConf = new XConfiguration(fs.open(xmlPath));
                String jobXmlConfString = XmlUtils.prettyPrint(jobXmlConf).toString();
                jobXmlConfString = XmlUtils.removeComments(jobXmlConfString);
                context.getELEvaluator().evaluate(jobXmlConfString, String.class);
            }
            return "OK";
        }
    } catch (WorkflowException ex) {
        throw new CommandException(ex);
    } catch (HadoopAccessorException ex) {
        throw new CommandException(ex);
    } catch (Exception ex) {
        throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
    }
}