Examples with LiteWorkflowInstance org.apache.oozie.workflow.lite.LiteWorkflowInstance used on opensource projects

Example 6 with LiteWorkflowInstance

use of org.apache.oozie.workflow.lite.LiteWorkflowInstance in project oozie by apache.

the class TestEventGeneration method testCoordinatorActionEvent.

@Test
public void testCoordinatorActionEvent() throws Exception {
    // avoid noise from other apptype events by setting it to only
    // coord action
    ehs.setAppTypes(new HashSet<String>(Arrays.asList("coordinator_action")));
    assertEquals(queue.size(), 0);
    Date startTime = DateUtils.parseDateOozieTZ("2013-01-01T10:00Z");
    Date endTime = DateUtils.parseDateOozieTZ("2013-01-01T10:01Z");
    CoordinatorJobBean coord = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0);
    modifyCoordForRunning(coord);
    // Action WAITING on materialization
    new CoordMaterializeTransitionXCommand(coord.getId(), 3600).call();
    final CoordActionGetJPAExecutor coordGetCmd = new CoordActionGetJPAExecutor(coord.getId() + "@1");
    CoordinatorActionBean action = jpaService.execute(coordGetCmd);
    assertEquals(CoordinatorAction.Status.WAITING, action.getStatus());
    assertEquals(1, queue.size());
    JobEvent event = (JobEvent) queue.poll();
    assertNotNull(event);
    assertEquals(EventStatus.WAITING, event.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, event.getAppType());
    assertEquals(action.getId(), event.getId());
    assertEquals(action.getJobId(), event.getParentId());
    assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime());
    assertNull(event.getStartTime());
    assertEquals(coord.getUser(), event.getUser());
    assertEquals(coord.getAppName(), event.getAppName());
    assertEquals(0, queue.size());
    // Make Action ready
    // In this case it will proceed to Running since n(ready_actions) < concurrency
    new CoordActionInputCheckXCommand(action.getId(), coord.getId()).call();
    action = jpaService.execute(coordGetCmd);
    assertEquals(CoordinatorAction.Status.RUNNING, action.getStatus());
    event = (JobEvent) queue.poll();
    assertEquals(EventStatus.STARTED, event.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, event.getAppType());
    assertEquals(action.getId(), event.getId());
    assertEquals(action.getJobId(), event.getParentId());
    assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime());
    WorkflowJobBean wfJob = jpaService.execute(new WorkflowJobGetJPAExecutor(action.getExternalId()));
    assertEquals(wfJob.getStartTime(), event.getStartTime());
    assertEquals(coord.getUser(), event.getUser());
    assertEquals(coord.getAppName(), event.getAppName());
    sleep(2000);
    // Action Success
    wfJob.setStatus(WorkflowJob.Status.SUCCEEDED);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfJob);
    action.setStatus(CoordinatorAction.Status.RUNNING);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    new CoordActionCheckXCommand(action.getId(), 0).call();
    action = jpaService.execute(coordGetCmd);
    assertEquals(CoordinatorAction.Status.SUCCEEDED, action.getStatus());
    List<Event> list = queue.pollBatch();
    event = (JobEvent) list.get(list.size() - 1);
    assertEquals(EventStatus.SUCCESS, event.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, event.getAppType());
    assertEquals(action.getId(), event.getId());
    assertEquals(action.getJobId(), event.getParentId());
    assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime());
    assertEquals(wfJob.getStartTime(), event.getStartTime());
    assertEquals(coord.getUser(), event.getUser());
    assertEquals(coord.getAppName(), event.getAppName());
    // Action Failure
    wfJob.setStatus(WorkflowJob.Status.FAILED);
    action.setStatus(CoordinatorAction.Status.RUNNING);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfJob);
    new CoordActionCheckXCommand(action.getId(), 0).call();
    action = jpaService.execute(coordGetCmd);
    assertEquals(CoordinatorAction.Status.FAILED, action.getStatus());
    event = (JobEvent) queue.poll();
    assertEquals(EventStatus.FAILURE, event.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, event.getAppType());
    assertEquals(action.getId(), event.getId());
    assertEquals(action.getJobId(), event.getParentId());
    assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime());
    assertEquals(wfJob.getStartTime(), event.getStartTime());
    assertEquals(coord.getUser(), event.getUser());
    assertEquals(coord.getAppName(), event.getAppName());
    // Action Suspended
    wfJob.setStatus(WorkflowJob.Status.SUSPENDED);
    action.setStatus(CoordinatorAction.Status.RUNNING);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfJob);
    new CoordActionCheckXCommand(action.getId(), 0).call();
    action = jpaService.execute(coordGetCmd);
    assertEquals(CoordinatorAction.Status.SUSPENDED, action.getStatus());
    event = (JobEvent) queue.poll();
    assertEquals(EventStatus.SUSPEND, event.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, event.getAppType());
    assertEquals(action.getId(), event.getId());
    assertEquals(action.getJobId(), event.getParentId());
    assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime());
    assertEquals(wfJob.getStartTime(), event.getStartTime());
    assertEquals(coord.getUser(), event.getUser());
    assertEquals(coord.getAppName(), event.getAppName());
    // Action start on Coord Resume
    coord.setStatus(CoordinatorJobBean.Status.SUSPENDED);
    CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB_STATUS, coord);
    action.setStatus(CoordinatorAction.Status.SUSPENDED);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    wfJob.setStatus(WorkflowJob.Status.SUSPENDED);
    WorkflowInstance wfInstance = wfJob.getWorkflowInstance();
    ((LiteWorkflowInstance) wfInstance).setStatus(WorkflowInstance.Status.SUSPENDED);
    wfJob.setWorkflowInstance(wfInstance);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MODIFIED, wfJob);
    queue.clear();
    new CoordResumeXCommand(coord.getId()).call();
    waitForEventGeneration(1000);
    CoordinatorActionEvent cevent = (CoordinatorActionEvent) queue.poll();
    assertEquals(EventStatus.STARTED, cevent.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, cevent.getAppType());
    assertEquals(action.getId(), cevent.getId());
    assertEquals(action.getJobId(), cevent.getParentId());
    assertEquals(action.getNominalTime(), cevent.getNominalTime());
    coord = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, coord.getId());
    assertEquals(coord.getLastModifiedTime(), cevent.getStartTime());
    // Action going to WAITING on Coord Rerun
    action.setStatus(CoordinatorAction.Status.KILLED);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    queue.clear();
    new CoordRerunXCommand(coord.getId(), RestConstants.JOB_COORD_SCOPE_ACTION, "1", false, true, false, null).call();
    waitFor(3 * 100, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return jpaService.execute(coordGetCmd).getStatus() == CoordinatorAction.Status.WAITING;
        }
    });
    cevent = (CoordinatorActionEvent) queue.poll();
    assertEquals(EventStatus.WAITING, cevent.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, cevent.getAppType());
    assertEquals(action.getId(), cevent.getId());
    assertEquals(action.getJobId(), cevent.getParentId());
    assertEquals(action.getNominalTime(), cevent.getNominalTime());
    assertEquals(wfJob.getStartTime(), event.getStartTime());
    assertNotNull(cevent.getMissingDeps());
}

Example 7 with LiteWorkflowInstance

use of org.apache.oozie.workflow.lite.LiteWorkflowInstance in project oozie by apache.

the class TestHCatELFunctions method testHCatTableExists.

@Test
public void testHCatTableExists() throws Exception {
    dropTable("db1", "table1", true);
    dropDatabase("db1", true);
    createDatabase("db1");
    createTable("db1", "table1");
    Configuration protoConf = new Configuration();
    protoConf.set(OozieClient.USER_NAME, getTestUser());
    protoConf.set("hadoop.job.ugi", getTestUser() + "," + "group");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, "appPath");
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set("test.dir", getTestCaseDir());
    conf.set("table1", getHCatURI("db1", "table1").toString());
    conf.set("table2", getHCatURI("db1", "table2").toString());
    LiteWorkflowApp def = new LiteWorkflowApp("name", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    LiteWorkflowInstance job = new LiteWorkflowInstance(def, conf, "wfId");
    WorkflowJobBean wf = new WorkflowJobBean();
    wf.setId(job.getId());
    wf.setAppName("name");
    wf.setAppPath("appPath");
    wf.setUser(getTestUser());
    wf.setGroup("group");
    wf.setWorkflowInstance(job);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    protoConf.writeXml(baos);
    wf.setProtoActionConf(baos.toString());
    WorkflowActionBean action = new WorkflowActionBean();
    action.setId("actionId");
    action.setName("actionName");
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, wf, action);
    assertEquals(true, (boolean) eval.evaluate("${hcat:exists(wf:conf('table1'))}", Boolean.class));
    assertEquals(false, (boolean) eval.evaluate("${hcat:exists(wf:conf('table2'))}", Boolean.class));
    dropTable("db1", "table1", true);
    dropDatabase("db1", true);
}

Example 8 with LiteWorkflowInstance

use of org.apache.oozie.workflow.lite.LiteWorkflowInstance in project oozie by apache.

the class SuspendXCommand method suspendJob.

/**
 * Suspend the workflow job and pending flag to false for the actions that are START_RETRY or START_MANUAL or
 * END_RETRY or END_MANUAL
 *
 * @param jpaService jpa service
 * @param workflow workflow job
 * @param id workflow job id
 * @param actionId workflow action id
 * @throws WorkflowException thrown if failed to suspend workflow instance
 * @throws CommandException thrown if unable set pending false for actions
 */
public static void suspendJob(JPAService jpaService, WorkflowJobBean workflow, String id, String actionId, List<UpdateEntry> updateList) throws WorkflowException, CommandException {
    if (workflow.getStatus() == WorkflowJob.Status.RUNNING) {
        workflow.getWorkflowInstance().suspend();
        WorkflowInstance wfInstance = workflow.getWorkflowInstance();
        ((LiteWorkflowInstance) wfInstance).setStatus(WorkflowInstance.Status.SUSPENDED);
        workflow.setStatus(WorkflowJob.Status.SUSPENDED);
        workflow.setWorkflowInstance(wfInstance);
        setPendingFalseForActions(jpaService, id, actionId, updateList);
        if (EventHandlerService.isEnabled()) {
            generateEvent(workflow);
        }
    }
}

Example 9 with LiteWorkflowInstance

use of org.apache.oozie.workflow.lite.LiteWorkflowInstance in project oozie by apache.

the class TestHadoopELFunctions method testELFunctionsReturningMapReduceStats.

public void testELFunctionsReturningMapReduceStats() throws Exception {
    String counters = "{\"g\":{\"c\":10},\"org.apache.hadoop.mapred.JobInProgress$Counter\":" + "{\"TOTAL_LAUNCHED_REDUCES\":1,\"TOTAL_LAUNCHED_MAPS\":2,\"DATA_LOCAL_MAPS\":2},\"ACTION_TYPE\":\"MAP_REDUCE\"," + "\"FileSystemCounters\":{\"FILE_BYTES_READ\":38,\"HDFS_BYTES_READ\":19," + "\"FILE_BYTES_WRITTEN\":146,\"HDFS_BYTES_WRITTEN\":16}," + "\"org.apache.hadoop.mapred.Task$Counter\":{\"REDUCE_INPUT_GROUPS\":2," + "\"COMBINE_OUTPUT_RECORDS\":0,\"MAP_INPUT_RECORDS\":2,\"REDUCE_SHUFFLE_BYTES\":22," + "\"REDUCE_OUTPUT_RECORDS\":2,\"SPILLED_RECORDS\":4,\"MAP_OUTPUT_BYTES\":28," + "\"MAP_INPUT_BYTES\":12,\"MAP_OUTPUT_RECORDS\":2,\"COMBINE_INPUT_RECORDS\":0," + "\"REDUCE_INPUT_RECORDS\":2}}";
    WorkflowJobBean workflow = new WorkflowJobBean();
    workflow.setProtoActionConf("<configuration/>");
    LiteWorkflowApp wfApp = new LiteWorkflowApp("x", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "a"));
    wfApp.addNode(new EndNodeDef("a", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    WorkflowInstance wi = new LiteWorkflowInstance(wfApp, new XConfiguration(), "1");
    workflow.setWorkflowInstance(wi);
    workflow.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.WORKFLOW));
    final WorkflowActionBean action = new WorkflowActionBean();
    action.setName("H");
    ActionXCommand.ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, action, false, false);
    context.setVar(MapReduceActionExecutor.HADOOP_COUNTERS, counters);
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, workflow, action);
    String group = "g";
    String name = "c";
    assertEquals(new Long(10), eval.evaluate("${hadoop:counters('H')['" + group + "']['" + name + "']}", Long.class));
    assertEquals(new Long(2), eval.evaluate("${hadoop:counters('H')[RECORDS][GROUPS]}", Long.class));
    assertEquals(new Long(2), eval.evaluate("${hadoop:counters('H')[RECORDS][REDUCE_IN]}", Long.class));
    assertEquals(new Long(2), eval.evaluate("${hadoop:counters('H')[RECORDS][REDUCE_OUT]}", Long.class));
    assertEquals(new Long(2), eval.evaluate("${hadoop:counters('H')[RECORDS][MAP_IN]}", Long.class));
    assertEquals(new Long(2), eval.evaluate("${hadoop:counters('H')[RECORDS][MAP_OUT]}", Long.class));
    assertEquals(ActionType.MAP_REDUCE.toString(), eval.evaluate("${hadoop:counters('H')['ACTION_TYPE']}", String.class));
}

Example 10 with LiteWorkflowInstance

use of org.apache.oozie.workflow.lite.LiteWorkflowInstance in project oozie by apache.

the class TestHadoopELFunctions method testELFunctionsReturningPigStats.

public void testELFunctionsReturningPigStats() throws Exception {
    String pigStats = "{\"ACTION_TYPE\":\"PIG\"," + "\"PIG_VERSION\":\"0.9.0\"," + "\"FEATURES\":\"UNKNOWN\"," + "\"ERROR_MESSAGE\":null," + "\"NUMBER_JOBS\":\"2\"," + "\"RECORD_WRITTEN\":\"33\"," + "\"JOB_GRAPH\":\"job_201111300933_0004,job_201111300933_0005\"," + "\"job_201111300933_0004\":{\"MAP_INPUT_RECORDS\":\"33\",\"MIN_REDUCE_TIME\":\"0\",\"MULTI_STORE_COUNTERS\":{}," + "\"ERROR_MESSAGE\":null,\"JOB_ID\":\"job_201111300933_0004\"}," + "\"job_201111300933_0005\":{\"MAP_INPUT_RECORDS\":\"37\",\"MIN_REDUCE_TIME\":\"0\",\"MULTI_STORE_COUNTERS\":{}," + "\"ERROR_MESSAGE\":null,\"JOB_ID\":\"job_201111300933_0005\"}," + "\"BYTES_WRITTEN\":\"1410\"," + "\"HADOOP_VERSION\":\"0.20.2\"," + "\"RETURN_CODE\":\"0\"," + "\"ERROR_CODE\":\"-1\"," + "}";
    WorkflowJobBean workflow = new WorkflowJobBean();
    workflow.setProtoActionConf("<configuration/>");
    LiteWorkflowApp wfApp = new LiteWorkflowApp("x", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "a"));
    wfApp.addNode(new EndNodeDef("a", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    WorkflowInstance wi = new LiteWorkflowInstance(wfApp, new XConfiguration(), "1");
    workflow.setWorkflowInstance(wi);
    workflow.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.WORKFLOW));
    final WorkflowActionBean action = new WorkflowActionBean();
    action.setName("H");
    ActionXCommand.ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, action, false, false);
    context.setVar(MapReduceActionExecutor.HADOOP_COUNTERS, pigStats);
    ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("workflow");
    DagELFunctions.configureEvaluator(eval, workflow, action);
    String version = "0.9.0";
    String jobGraph = "job_201111300933_0004,job_201111300933_0005";
    HashMap<String, String> job1StatusMap = new HashMap<String, String>();
    job1StatusMap.put("\"MAP_INPUT_RECORDS\"", "\"33\"");
    job1StatusMap.put("\"MIN_REDUCE_TIME\"", "\"0\"");
    job1StatusMap.put("\"MULTI_STORE_COUNTERS\"", "{}");
    job1StatusMap.put("\"ERROR_MESSAGE\"", "null");
    job1StatusMap.put("\"JOB_ID\"", "\"job_201111300933_0004\"");
    HashMap<String, String> job2StatusMap = new HashMap<String, String>();
    job2StatusMap.put("\"MAP_INPUT_RECORDS\"", "\"37\"");
    job2StatusMap.put("\"MIN_REDUCE_TIME\"", "\"0\"");
    job2StatusMap.put("\"MULTI_STORE_COUNTERS\"", "{}");
    job2StatusMap.put("\"ERROR_MESSAGE\"", "null");
    job2StatusMap.put("\"JOB_ID\"", "\"job_201111300933_0005\"");
    assertEquals(ActionType.PIG.toString(), eval.evaluate("${hadoop:counters('H')['ACTION_TYPE']}", String.class));
    assertEquals(version, eval.evaluate("${hadoop:counters('H')['PIG_VERSION']}", String.class));
    assertEquals(jobGraph, eval.evaluate("${hadoop:counters('H')['JOB_GRAPH']}", String.class));
    String[] jobStatusItems = { "\"MAP_INPUT_RECORDS\"", "\"MIN_REDUCE_TIME\"", "\"MULTI_STORE_COUNTERS\"", "\"ERROR_MESSAGE\"", "\"JOB_ID\"" };
    String job1StatusResult = eval.evaluate("${hadoop:counters('H')['job_201111300933_0004']}", String.class);
    job1StatusResult = job1StatusResult.substring(job1StatusResult.indexOf('{') + 1, job1StatusResult.lastIndexOf('}'));
    String[] job1StatusResArray = job1StatusResult.split(",");
    HashMap<String, String> job1StatusResMap = new HashMap<String, String>();
    for (String status : job1StatusResArray) {
        String[] tmp = status.split(":");
        job1StatusResMap.put(tmp[0], tmp[1]);
    }
    for (String item : jobStatusItems) {
        assertEquals(job1StatusMap.get(item), job1StatusResMap.get(item));
    }
    String job2StatusResult = eval.evaluate("${hadoop:counters('H')['job_201111300933_0005']}", String.class);
    job2StatusResult = job2StatusResult.substring(job2StatusResult.indexOf('{') + 1, job2StatusResult.lastIndexOf('}'));
    String[] job2StatusResArray = job2StatusResult.split(",");
    HashMap<String, String> job2StatusResMap = new HashMap<String, String>();
    for (String status : job2StatusResArray) {
        String[] tmp = status.split(":");
        job2StatusResMap.put(tmp[0], tmp[1]);
    }
    for (String item : jobStatusItems) {
        assertEquals(job2StatusMap.get(item), job2StatusResMap.get(item));
    }
    assertEquals(new Long(33), eval.evaluate("${hadoop:counters('H')['job_201111300933_0004']['MAP_INPUT_RECORDS']}", Long.class));
}