Examples with ELEvaluator org.apache.oozie.util.ELEvaluator used on opensource projects

Example 61 with ELEvaluator

use of org.apache.oozie.util.ELEvaluator in project oozie by apache.

the class HCatELFunctions method ph3_coord_dataInPartitions.

/**
 * Used to specify the entire HCat partition defining input for workflow job. <p> Look for two evaluator-level
 * variables <p> A) .datain.&lt;DATAIN_NAME&gt; B) .datain.&lt;DATAIN_NAME&gt;.unresolved <p> A defines the data-in HCat URI.
 * <p> B defines whether there are any unresolved EL-function (i.e latest) <p> If there are something unresolved,
 * this function will echo back the original function <p> otherwise it sends the partition.
 *
 * @param dataInName : DataIn name
 * @param type : for action type: hive-export
 */
public static String ph3_coord_dataInPartitions(String dataInName, String type) {
    ELEvaluator eval = ELEvaluator.getCurrent();
    String uri = (String) eval.getVariable(".datain." + dataInName);
    Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
    if (unresolved != null && unresolved.booleanValue() == true) {
        return "${coord:dataInPartitions('" + dataInName + "', '" + type + "')}";
    }
    String partitionValue = null;
    if (uri != null) {
        if (type.equals("hive-export")) {
            String[] uriList = HCatURIParser.splitHCatUris(uri, HCAT_URI_PATTERN);
            if (uriList.length > 1) {
                throw new RuntimeException("Multiple partitions not supported for hive-export type. Dataset name: " + dataInName + " URI: " + uri);
            }
            try {
                partitionValue = new HCatURI(uri).toPartitionValueString(type);
            } catch (URISyntaxException e) {
                throw new RuntimeException("Parsing exception for HCatURI " + uri, e);
            }
        } else {
            throw new RuntimeException("Unsupported type: " + type + " dataset name: " + dataInName);
        }
    } else {
        XLog.getLog(HCatELFunctions.class).warn("URI is null");
        return null;
    }
    return partitionValue;
}

Example 62 with ELEvaluator

use of org.apache.oozie.util.ELEvaluator in project oozie by apache.

the class HCatELFunctions method ph3_coord_dataOutPartitions.

/**
 * Used to specify the entire HCat partition defining output for workflow job.<p> Look for two evaluator-level
 * variables <p> A) .dataout.&lt;DATAOUT_NAME&gt; B) .dataout.&lt;DATAOUT_NAME&gt;.unresolved <p> A defines the data-out
 * HCat URI. <p> B defines whether there are any unresolved EL-function (i.e latest) <p> If there are something
 * unresolved, this function will echo back the original function <p> otherwise it sends the partition.
 *
 * @param dataOutName : DataOut name
 */
public static String ph3_coord_dataOutPartitions(String dataOutName) {
    ELEvaluator eval = ELEvaluator.getCurrent();
    String uri = (String) eval.getVariable(".dataout." + dataOutName);
    Boolean unresolved = (Boolean) eval.getVariable(".dataout." + dataOutName + ".unresolved");
    if (unresolved != null && unresolved.booleanValue() == true) {
        return "${coord:dataOutPartitions('" + dataOutName + "')}";
    }
    try {
        return new HCatURI(uri).toPartitionString();
    } catch (URISyntaxException e) {
        throw new RuntimeException("Parsing exception for HCatURI " + uri + ". details: " + e);
    }
}

Example 63 with ELEvaluator

use of org.apache.oozie.util.ELEvaluator in project oozie by apache.

the class TestFsELFunctions method testFunctions.

public void testFunctions() throws Exception {
    String file1 = new Path(getFsTestCaseDir(), "file1").toString();
    String file2 = new Path(getFsTestCaseDir(), "file2").toString();
    String dir = new Path(getFsTestCaseDir(), "dir").toString();
    Configuration protoConf = new Configuration();
    protoConf.set(OozieClient.USER_NAME, getTestUser());
    protoConf.set("hadoop.job.ugi", getTestUser() + "," + "group");
    FileSystem fs = getFileSystem();
    fs.mkdirs(new Path(dir));
    fs.create(new Path(file1)).close();
    OutputStream os = fs.create(new Path(dir, "a"));
    byte[] arr = new byte[1];
    os.write(arr);
    os.close();
    os = fs.create(new Path(dir, "b"));
    arr = new byte[2];
    os.write(arr);
    os.close();
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, "appPath");
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set("test.dir", getTestCaseDir());
    conf.set("file1", file1);
    conf.set("file2", file2);
    conf.set("file3", "${file2}");
    conf.set("file4", getFsTestCaseDir() + "/file{1,2}");
    conf.set("file5", getFsTestCaseDir() + "/file*");
    conf.set("file6", getFsTestCaseDir() + "/file_*");
    conf.set("dir", dir);
    LiteWorkflowApp def = new LiteWorkflowApp("name", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    LiteWorkflowInstance job = new LiteWorkflowInstance(def, conf, "wfId");
    WorkflowJobBean wf = new WorkflowJobBean();
    wf.setId(job.getId());
    wf.setAppName("name");
    wf.setAppPath("appPath");
    wf.setUser(getTestUser());
    wf.setGroup("group");
    wf.setWorkflowInstance(job);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    protoConf.writeXml(baos);
    wf.setProtoActionConf(baos.toString());
    WorkflowActionBean action = new WorkflowActionBean();
    action.setId("actionId");
    action.setName("actionName");
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, wf, action);
    assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('file1'))}", Boolean.class));
    assertEquals(false, (boolean) eval.evaluate("${fs:exists(wf:conf('file2'))}", Boolean.class));
    assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('file4'))}", Boolean.class));
    assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('file5'))}", Boolean.class));
    assertEquals(false, (boolean) eval.evaluate("${fs:exists(wf:conf('file6'))}", Boolean.class));
    assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('dir'))}", Boolean.class));
    assertEquals(false, (boolean) eval.evaluate("${fs:isDir(wf:conf('file1'))}", Boolean.class));
    assertEquals(0, (int) eval.evaluate("${fs:fileSize(wf:conf('file1'))}", Integer.class));
    assertEquals(-1, (int) eval.evaluate("${fs:fileSize(wf:conf('file2'))}", Integer.class));
    assertEquals(3, (int) eval.evaluate("${fs:dirSize(wf:conf('dir'))}", Integer.class));
    assertEquals(-1, (int) eval.evaluate("${fs:blockSize(wf:conf('file2'))}", Integer.class));
    assertTrue(eval.evaluate("${fs:blockSize(wf:conf('file1'))}", Integer.class) > 0);
}

Example 64 with ELEvaluator

use of org.apache.oozie.util.ELEvaluator in project oozie by apache.

the class TestDagELFunctions method testFunctions.

public void testFunctions() throws Exception {
    XConfiguration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, "appPath");
    conf.set(OozieClient.USER_NAME, "user");
    conf.set("a", "A");
    LiteWorkflowApp def = new LiteWorkflowApp("name", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    LiteWorkflowInstance job = new LiteWorkflowInstance(def, conf, "wfId");
    WorkflowJobBean wf = new WorkflowJobBean();
    wf.setId(job.getId());
    wf.setAppName("name");
    wf.setAppPath("appPath");
    wf.setUser("user");
    wf.setGroup("group");
    wf.setWorkflowInstance(job);
    wf.setRun(2);
    wf.setProtoActionConf(conf.toXmlString());
    WorkflowActionBean action = new WorkflowActionBean();
    action.setId("actionId");
    action.setName("actionName");
    action.setErrorInfo("ec", "em");
    action.setData("b=B");
    action.setExternalId("ext");
    action.setTrackerUri("tracker");
    action.setExternalStatus("externalStatus");
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, wf, action);
    assertEquals("wfId", eval.evaluate("${wf:id()}", String.class));
    assertEquals("name", eval.evaluate("${wf:name()}", String.class));
    assertEquals("appPath", eval.evaluate("${wf:appPath()}", String.class));
    assertEquals("A", eval.evaluate("${wf:conf('a')}", String.class));
    assertEquals("A", eval.evaluate("${a}", String.class));
    assertEquals("user", eval.evaluate("${wf:user()}", String.class));
    assertEquals("group", eval.evaluate("${wf:group()}", String.class));
    assertTrue(eval.evaluate("${wf:callback('XX')}", String.class).contains("id=actionId"));
    assertTrue(eval.evaluate("${wf:callback('XX')}", String.class).contains("status=XX"));
    assertTrue(eval.evaluate("${wf:callback('XX')}", String.class).contains("status=XX"));
    assertEquals(2, (int) eval.evaluate("${wf:run()}", Integer.class));
    action.setStatus(WorkflowAction.Status.ERROR);
    System.out.println("WorkflowInstance " + wf.getWorkflowInstance().getStatus().toString());
    WorkflowInstance wfInstance = wf.getWorkflowInstance();
    DagELFunctions.setActionInfo(wfInstance, action);
    wf.setWorkflowInstance(wfInstance);
    assertEquals("actionName", eval.evaluate("${wf:lastErrorNode()}", String.class));
    assertEquals("ec", eval.evaluate("${wf:errorCode('actionName')}", String.class));
    assertEquals("em", eval.evaluate("${wf:errorMessage('actionName')}", String.class));
    assertEquals("B", eval.evaluate("${wf:actionData('actionName')['b']}", String.class));
    String expected = XmlUtils.escapeCharsForXML("{\"b\":\"B\"}");
    assertEquals(expected, eval.evaluate("${toJsonStr(wf:actionData('actionName'))}", String.class));
    expected = XmlUtils.escapeCharsForXML("b=B");
    assertTrue(eval.evaluate("${toPropertiesStr(wf:actionData('actionName'))}", String.class).contains(expected));
    conf = new XConfiguration();
    conf.set("b", "B");
    expected = XmlUtils.escapeCharsForXML(XmlUtils.prettyPrint(conf).toString());
    assertTrue(eval.evaluate("${toConfigurationStr(wf:actionData('actionName'))}", String.class).contains(expected));
    assertEquals("ext", eval.evaluate("${wf:actionExternalId('actionName')}", String.class));
    assertEquals("tracker", eval.evaluate("${wf:actionTrackerUri('actionName')}", String.class));
    assertEquals("externalStatus", eval.evaluate("${wf:actionExternalStatus('actionName')}", String.class));
}

Example 65 with ELEvaluator

use of org.apache.oozie.util.ELEvaluator in project oozie by apache.

the class TestDagELFunctions method testLastErrorNodeWithRetryFail.

// This test simulates an action that gets retried because of an Error and never succeeds on later retries.  The lastErrorNode
// EL function should be set to this node, but only after the last retry.
public void testLastErrorNodeWithRetryFail() throws Exception {
    WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.END_RETRY, true);
    action.setType("java");
    action.setExternalStatus("FAILED");
    action.setErrorInfo("JA018", "some error occurred");
    WorkflowActionQueryExecutor.getInstance().executeUpdate(WorkflowActionQueryExecutor.WorkflowActionQuery.UPDATE_ACTION, action);
    new ActionEndXCommandIgnoreSignalException(action.getId(), action.getType()).call();
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    job = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQueryExecutor.WorkflowJobQuery.GET_WORKFLOW, job.getId());
    action = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQueryExecutor.WorkflowActionQuery.GET_ACTION, action.getId());
    DagELFunctions.configureEvaluator(eval, job, action);
    assertEquals("", eval.evaluate("${wf:lastErrorNode()}", String.class));
    action.setExternalStatus("FAILED");
    action.setErrorInfo("JA018", "some error occurred");
    action.setStatus(WorkflowAction.Status.END_RETRY);
    // make it the last retry
    action.setUserRetryCount(action.getUserRetryMax());
    WorkflowActionQueryExecutor.getInstance().executeUpdate(WorkflowActionQueryExecutor.WorkflowActionQuery.UPDATE_ACTION, action);
    new ActionEndXCommandIgnoreSignalException(action.getId(), action.getType()).call();
    job = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQueryExecutor.WorkflowJobQuery.GET_WORKFLOW, job.getId());
    action = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQueryExecutor.WorkflowActionQuery.GET_ACTION, action.getId());
    assertEquals(WorkflowAction.Status.ERROR, action.getStatus());
    DagELFunctions.configureEvaluator(eval, job, action);
    assertEquals(action.getName(), eval.evaluate("${wf:lastErrorNode()}", String.class));
}