Examples with StartNodeDef org.apache.oozie.workflow.lite.StartNodeDef used on opensource projects

Example 11 with StartNodeDef

use of org.apache.oozie.workflow.lite.StartNodeDef in project oozie by apache.

the class TestHadoopELFunctions method testELFunctionsReturningMapReduceStats.

public void testELFunctionsReturningMapReduceStats() throws Exception {
    String counters = "{\"g\":{\"c\":10},\"org.apache.hadoop.mapred.JobInProgress$Counter\":" + "{\"TOTAL_LAUNCHED_REDUCES\":1,\"TOTAL_LAUNCHED_MAPS\":2,\"DATA_LOCAL_MAPS\":2},\"ACTION_TYPE\":\"MAP_REDUCE\"," + "\"FileSystemCounters\":{\"FILE_BYTES_READ\":38,\"HDFS_BYTES_READ\":19," + "\"FILE_BYTES_WRITTEN\":146,\"HDFS_BYTES_WRITTEN\":16}," + "\"org.apache.hadoop.mapred.Task$Counter\":{\"REDUCE_INPUT_GROUPS\":2," + "\"COMBINE_OUTPUT_RECORDS\":0,\"MAP_INPUT_RECORDS\":2,\"REDUCE_SHUFFLE_BYTES\":22," + "\"REDUCE_OUTPUT_RECORDS\":2,\"SPILLED_RECORDS\":4,\"MAP_OUTPUT_BYTES\":28," + "\"MAP_INPUT_BYTES\":12,\"MAP_OUTPUT_RECORDS\":2,\"COMBINE_INPUT_RECORDS\":0," + "\"REDUCE_INPUT_RECORDS\":2}}";
    WorkflowJobBean workflow = new WorkflowJobBean();
    workflow.setProtoActionConf("<configuration/>");
    LiteWorkflowApp wfApp = new LiteWorkflowApp("x", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "a"));
    wfApp.addNode(new EndNodeDef("a", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    WorkflowInstance wi = new LiteWorkflowInstance(wfApp, new XConfiguration(), "1");
    workflow.setWorkflowInstance(wi);
    workflow.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.WORKFLOW));
    final WorkflowActionBean action = new WorkflowActionBean();
    action.setName("H");
    ActionXCommand.ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, action, false, false);
    context.setVar(MapReduceActionExecutor.HADOOP_COUNTERS, counters);
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, workflow, action);
    String group = "g";
    String name = "c";
    assertEquals(new Long(10), eval.evaluate("${hadoop:counters('H')['" + group + "']['" + name + "']}", Long.class));
    assertEquals(new Long(2), eval.evaluate("${hadoop:counters('H')[RECORDS][GROUPS]}", Long.class));
    assertEquals(new Long(2), eval.evaluate("${hadoop:counters('H')[RECORDS][REDUCE_IN]}", Long.class));
    assertEquals(new Long(2), eval.evaluate("${hadoop:counters('H')[RECORDS][REDUCE_OUT]}", Long.class));
    assertEquals(new Long(2), eval.evaluate("${hadoop:counters('H')[RECORDS][MAP_IN]}", Long.class));
    assertEquals(new Long(2), eval.evaluate("${hadoop:counters('H')[RECORDS][MAP_OUT]}", Long.class));
    assertEquals(ActionType.MAP_REDUCE.toString(), eval.evaluate("${hadoop:counters('H')['ACTION_TYPE']}", String.class));
}

Example 12 with StartNodeDef

use of org.apache.oozie.workflow.lite.StartNodeDef in project oozie by apache.

the class TestHadoopELFunctions method testELFunctionsReturningPigStats.

public void testELFunctionsReturningPigStats() throws Exception {
    String pigStats = "{\"ACTION_TYPE\":\"PIG\"," + "\"PIG_VERSION\":\"0.9.0\"," + "\"FEATURES\":\"UNKNOWN\"," + "\"ERROR_MESSAGE\":null," + "\"NUMBER_JOBS\":\"2\"," + "\"RECORD_WRITTEN\":\"33\"," + "\"JOB_GRAPH\":\"job_201111300933_0004,job_201111300933_0005\"," + "\"job_201111300933_0004\":{\"MAP_INPUT_RECORDS\":\"33\",\"MIN_REDUCE_TIME\":\"0\",\"MULTI_STORE_COUNTERS\":{}," + "\"ERROR_MESSAGE\":null,\"JOB_ID\":\"job_201111300933_0004\"}," + "\"job_201111300933_0005\":{\"MAP_INPUT_RECORDS\":\"37\",\"MIN_REDUCE_TIME\":\"0\",\"MULTI_STORE_COUNTERS\":{}," + "\"ERROR_MESSAGE\":null,\"JOB_ID\":\"job_201111300933_0005\"}," + "\"BYTES_WRITTEN\":\"1410\"," + "\"HADOOP_VERSION\":\"0.20.2\"," + "\"RETURN_CODE\":\"0\"," + "\"ERROR_CODE\":\"-1\"," + "}";
    WorkflowJobBean workflow = new WorkflowJobBean();
    workflow.setProtoActionConf("<configuration/>");
    LiteWorkflowApp wfApp = new LiteWorkflowApp("x", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "a"));
    wfApp.addNode(new EndNodeDef("a", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    WorkflowInstance wi = new LiteWorkflowInstance(wfApp, new XConfiguration(), "1");
    workflow.setWorkflowInstance(wi);
    workflow.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.WORKFLOW));
    final WorkflowActionBean action = new WorkflowActionBean();
    action.setName("H");
    ActionXCommand.ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, action, false, false);
    context.setVar(MapReduceActionExecutor.HADOOP_COUNTERS, pigStats);
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, workflow, action);
    String version = "0.9.0";
    String jobGraph = "job_201111300933_0004,job_201111300933_0005";
    HashMap<String, String> job1StatusMap = new HashMap<String, String>();
    job1StatusMap.put("\"MAP_INPUT_RECORDS\"", "\"33\"");
    job1StatusMap.put("\"MIN_REDUCE_TIME\"", "\"0\"");
    job1StatusMap.put("\"MULTI_STORE_COUNTERS\"", "{}");
    job1StatusMap.put("\"ERROR_MESSAGE\"", "null");
    job1StatusMap.put("\"JOB_ID\"", "\"job_201111300933_0004\"");
    HashMap<String, String> job2StatusMap = new HashMap<String, String>();
    job2StatusMap.put("\"MAP_INPUT_RECORDS\"", "\"37\"");
    job2StatusMap.put("\"MIN_REDUCE_TIME\"", "\"0\"");
    job2StatusMap.put("\"MULTI_STORE_COUNTERS\"", "{}");
    job2StatusMap.put("\"ERROR_MESSAGE\"", "null");
    job2StatusMap.put("\"JOB_ID\"", "\"job_201111300933_0005\"");
    assertEquals(ActionType.PIG.toString(), eval.evaluate("${hadoop:counters('H')['ACTION_TYPE']}", String.class));
    assertEquals(version, eval.evaluate("${hadoop:counters('H')['PIG_VERSION']}", String.class));
    assertEquals(jobGraph, eval.evaluate("${hadoop:counters('H')['JOB_GRAPH']}", String.class));
    String[] jobStatusItems = { "\"MAP_INPUT_RECORDS\"", "\"MIN_REDUCE_TIME\"", "\"MULTI_STORE_COUNTERS\"", "\"ERROR_MESSAGE\"", "\"JOB_ID\"" };
    String job1StatusResult = eval.evaluate("${hadoop:counters('H')['job_201111300933_0004']}", String.class);
    job1StatusResult = job1StatusResult.substring(job1StatusResult.indexOf('{') + 1, job1StatusResult.lastIndexOf('}'));
    String[] job1StatusResArray = job1StatusResult.split(",");
    HashMap<String, String> job1StatusResMap = new HashMap<String, String>();
    for (String status : job1StatusResArray) {
        String[] tmp = status.split(":");
        job1StatusResMap.put(tmp[0], tmp[1]);
    }
    for (String item : jobStatusItems) {
        assertEquals(job1StatusMap.get(item), job1StatusResMap.get(item));
    }
    String job2StatusResult = eval.evaluate("${hadoop:counters('H')['job_201111300933_0005']}", String.class);
    job2StatusResult = job2StatusResult.substring(job2StatusResult.indexOf('{') + 1, job2StatusResult.lastIndexOf('}'));
    String[] job2StatusResArray = job2StatusResult.split(",");
    HashMap<String, String> job2StatusResMap = new HashMap<String, String>();
    for (String status : job2StatusResArray) {
        String[] tmp = status.split(":");
        job2StatusResMap.put(tmp[0], tmp[1]);
    }
    for (String item : jobStatusItems) {
        assertEquals(job2StatusMap.get(item), job2StatusResMap.get(item));
    }
    assertEquals(new Long(33), eval.evaluate("${hadoop:counters('H')['job_201111300933_0004']['MAP_INPUT_RECORDS']}", Long.class));
}

Example 13 with StartNodeDef

use of org.apache.oozie.workflow.lite.StartNodeDef in project oozie by apache.

the class TestHadoopELFunctions method testHadoopConfFunctions.

public void testHadoopConfFunctions() throws Exception {
    XConfiguration jobConf = new XConfiguration();
    XConfiguration.copy(createJobConf(), jobConf);
    String testHadoopOptionValue = jobConf.get("mapred.tasktracker.map.tasks.maximum");
    jobConf.set("test.name.node.uri", getNameNodeUri());
    jobConf.set("test.hadoop.option", "mapred.tasktracker.map.tasks.maximum");
    WorkflowJobBean workflow = new WorkflowJobBean();
    workflow.setProtoActionConf("<configuration/>");
    LiteWorkflowApp wfApp = new LiteWorkflowApp("x", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "a"));
    wfApp.addNode(new EndNodeDef("a", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    WorkflowInstance wi = new LiteWorkflowInstance(wfApp, jobConf, "1");
    workflow.setWorkflowInstance(wi);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    jobConf.writeXml(baos);
    workflow.setProtoActionConf(baos.toString());
    final WorkflowActionBean action = new WorkflowActionBean();
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, workflow, action);
    assertEquals(testHadoopOptionValue, eval.evaluate("${hadoop:conf(wf:conf('test.name.node.uri'), wf:conf('test.hadoop.option'))}", String.class));
}

Example 14 with StartNodeDef

use of org.apache.oozie.workflow.lite.StartNodeDef in project oozie by apache.

the class TestFsELFunctions method testFunctions.

public void testFunctions() throws Exception {
    String file1 = new Path(getFsTestCaseDir(), "file1").toString();
    String file2 = new Path(getFsTestCaseDir(), "file2").toString();
    String dir = new Path(getFsTestCaseDir(), "dir").toString();
    Configuration protoConf = new Configuration();
    protoConf.set(OozieClient.USER_NAME, getTestUser());
    protoConf.set("hadoop.job.ugi", getTestUser() + "," + "group");
    FileSystem fs = getFileSystem();
    fs.mkdirs(new Path(dir));
    fs.create(new Path(file1)).close();
    OutputStream os = fs.create(new Path(dir, "a"));
    byte[] arr = new byte[1];
    os.write(arr);
    os.close();
    os = fs.create(new Path(dir, "b"));
    arr = new byte[2];
    os.write(arr);
    os.close();
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, "appPath");
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set("test.dir", getTestCaseDir());
    conf.set("file1", file1);
    conf.set("file2", file2);
    conf.set("file3", "${file2}");
    conf.set("file4", getFsTestCaseDir() + "/file{1,2}");
    conf.set("file5", getFsTestCaseDir() + "/file*");
    conf.set("file6", getFsTestCaseDir() + "/file_*");
    conf.set("dir", dir);
    LiteWorkflowApp def = new LiteWorkflowApp("name", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    LiteWorkflowInstance job = new LiteWorkflowInstance(def, conf, "wfId");
    WorkflowJobBean wf = new WorkflowJobBean();
    wf.setId(job.getId());
    wf.setAppName("name");
    wf.setAppPath("appPath");
    wf.setUser(getTestUser());
    wf.setGroup("group");
    wf.setWorkflowInstance(job);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    protoConf.writeXml(baos);
    wf.setProtoActionConf(baos.toString());
    WorkflowActionBean action = new WorkflowActionBean();
    action.setId("actionId");
    action.setName("actionName");
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, wf, action);
    assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('file1'))}", Boolean.class));
    assertEquals(false, (boolean) eval.evaluate("${fs:exists(wf:conf('file2'))}", Boolean.class));
    assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('file4'))}", Boolean.class));
    assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('file5'))}", Boolean.class));
    assertEquals(false, (boolean) eval.evaluate("${fs:exists(wf:conf('file6'))}", Boolean.class));
    assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('dir'))}", Boolean.class));
    assertEquals(false, (boolean) eval.evaluate("${fs:isDir(wf:conf('file1'))}", Boolean.class));
    assertEquals(0, (int) eval.evaluate("${fs:fileSize(wf:conf('file1'))}", Integer.class));
    assertEquals(-1, (int) eval.evaluate("${fs:fileSize(wf:conf('file2'))}", Integer.class));
    assertEquals(3, (int) eval.evaluate("${fs:dirSize(wf:conf('dir'))}", Integer.class));
    assertEquals(-1, (int) eval.evaluate("${fs:blockSize(wf:conf('file2'))}", Integer.class));
    assertTrue(eval.evaluate("${fs:blockSize(wf:conf('file1'))}", Integer.class) > 0);
}

Example 15 with StartNodeDef

use of org.apache.oozie.workflow.lite.StartNodeDef in project oozie by apache.

the class TestDagELFunctions method testFunctions.

public void testFunctions() throws Exception {
    XConfiguration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, "appPath");
    conf.set(OozieClient.USER_NAME, "user");
    conf.set("a", "A");
    LiteWorkflowApp def = new LiteWorkflowApp("name", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    LiteWorkflowInstance job = new LiteWorkflowInstance(def, conf, "wfId");
    WorkflowJobBean wf = new WorkflowJobBean();
    wf.setId(job.getId());
    wf.setAppName("name");
    wf.setAppPath("appPath");
    wf.setUser("user");
    wf.setGroup("group");
    wf.setWorkflowInstance(job);
    wf.setRun(2);
    wf.setProtoActionConf(conf.toXmlString());
    WorkflowActionBean action = new WorkflowActionBean();
    action.setId("actionId");
    action.setName("actionName");
    action.setErrorInfo("ec", "em");
    action.setData("b=B");
    action.setExternalId("ext");
    action.setTrackerUri("tracker");
    action.setExternalStatus("externalStatus");
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, wf, action);
    assertEquals("wfId", eval.evaluate("${wf:id()}", String.class));
    assertEquals("name", eval.evaluate("${wf:name()}", String.class));
    assertEquals("appPath", eval.evaluate("${wf:appPath()}", String.class));
    assertEquals("A", eval.evaluate("${wf:conf('a')}", String.class));
    assertEquals("A", eval.evaluate("${a}", String.class));
    assertEquals("user", eval.evaluate("${wf:user()}", String.class));
    assertEquals("group", eval.evaluate("${wf:group()}", String.class));
    assertTrue(eval.evaluate("${wf:callback('XX')}", String.class).contains("id=actionId"));
    assertTrue(eval.evaluate("${wf:callback('XX')}", String.class).contains("status=XX"));
    assertTrue(eval.evaluate("${wf:callback('XX')}", String.class).contains("status=XX"));
    assertEquals(2, (int) eval.evaluate("${wf:run()}", Integer.class));
    action.setStatus(WorkflowAction.Status.ERROR);
    System.out.println("WorkflowInstance " + wf.getWorkflowInstance().getStatus().toString());
    WorkflowInstance wfInstance = wf.getWorkflowInstance();
    DagELFunctions.setActionInfo(wfInstance, action);
    wf.setWorkflowInstance(wfInstance);
    assertEquals("actionName", eval.evaluate("${wf:lastErrorNode()}", String.class));
    assertEquals("ec", eval.evaluate("${wf:errorCode('actionName')}", String.class));
    assertEquals("em", eval.evaluate("${wf:errorMessage('actionName')}", String.class));
    assertEquals("B", eval.evaluate("${wf:actionData('actionName')['b']}", String.class));
    String expected = XmlUtils.escapeCharsForXML("{\"b\":\"B\"}");
    assertEquals(expected, eval.evaluate("${toJsonStr(wf:actionData('actionName'))}", String.class));
    expected = XmlUtils.escapeCharsForXML("b=B");
    assertTrue(eval.evaluate("${toPropertiesStr(wf:actionData('actionName'))}", String.class).contains(expected));
    conf = new XConfiguration();
    conf.set("b", "B");
    expected = XmlUtils.escapeCharsForXML(XmlUtils.prettyPrint(conf).toString());
    assertTrue(eval.evaluate("${toConfigurationStr(wf:actionData('actionName'))}", String.class).contains(expected));
    assertEquals("ext", eval.evaluate("${wf:actionExternalId('actionName')}", String.class));
    assertEquals("tracker", eval.evaluate("${wf:actionTrackerUri('actionName')}", String.class));
    assertEquals("externalStatus", eval.evaluate("${wf:actionExternalStatus('actionName')}", String.class));
}