Examples with CoordMaterializeTransitionXCommand org.apache.oozie.command.coord.CoordMaterializeTransitionXCommand used on opensource projects

Example 6 with CoordMaterializeTransitionXCommand

use of org.apache.oozie.command.coord.CoordMaterializeTransitionXCommand in project oozie by apache.

the class TestCoordinatorInputLogic method startCoordActionForWaiting.

private void startCoordActionForWaiting(final String jobId) throws CommandException, JPAExecutorException, JDOMException {
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
    waitFor(5 * 1000, new Predicate() {

        public boolean evaluate() throws Exception {
            CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get(CoordActionQuery.GET_COORD_ACTION, jobId + "@1");
            return !actionBean.getStatus().equals(CoordinatorAction.Status.WAITING);
        }
    });
    CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get(CoordActionQuery.GET_COORD_ACTION, jobId + "@1");
    assertTrue("should be waiting", actionBean.getStatus().equals(CoordinatorAction.Status.WAITING));
}

Example 7 with CoordMaterializeTransitionXCommand

use of org.apache.oozie.command.coord.CoordMaterializeTransitionXCommand in project oozie by apache.

the class TestCoordinatorInputLogic method testCombineWithMin.

public void testCombineWithMin() throws Exception {
    Configuration conf = getConf();
    String inputLogic = // @formatter:off
    "<combine name=\"test\" min=\"4\">" + "<data-in dataset=\"A\" />" + "<data-in dataset=\"B\" />" + "</combine>";
    // @formatter:on
    conf.set("initial_instance_a", "2014-10-07T00:00Z");
    conf.set("initial_instance_b", "2014-10-07T00:00Z");
    final String jobId = _testCoordSubmit("coord-inputlogic-range.xml", conf, inputLogic, getInputEventForRange());
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    String input1 = createTestCaseSubDir("input-data/a/2014/10/08/00/_SUCCESS".split("/"));
    String input2 = createTestCaseSubDir("input-data/a/2014/10/07/23/_SUCCESS".split("/"));
    String input3 = createTestCaseSubDir("input-data/a/2014/10/07/22/_SUCCESS".split("/"));
    String input4 = createTestCaseSubDir("input-data/b/2014/10/07/21/_SUCCESS".split("/"));
    String input5 = createTestCaseSubDir("input-data/b/2014/10/07/20/_SUCCESS".split("/"));
    startCoordAction(jobId);
    CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get(CoordActionQuery.GET_COORD_ACTION, jobId + "@1");
    assertFalse(CoordinatorAction.Status.WAITING.equals(actionBean.getStatus()));
    XConfiguration runConf = new XConfiguration(new StringReader(actionBean.getRunConf()));
    String dataSets = runConf.get("inputLogicData");
    assertEquals(dataSets.split(",").length, 5);
    checkDataSets(dataSets, input1, input2, input3, input4, input5);
}

Example 8 with CoordMaterializeTransitionXCommand

use of org.apache.oozie.command.coord.CoordMaterializeTransitionXCommand in project oozie by apache.

the class TestEventGeneration method testForNoDuplicatesCoordinatorActionEvents.

@Test
public void testForNoDuplicatesCoordinatorActionEvents() throws Exception {
    // test coordinator action events (failure case)
    Date startTime = DateUtils.parseDateOozieTZ("2009-02-01T23:59Z");
    Date endTime = DateUtils.parseDateOozieTZ("2009-02-02T23:59Z");
    CoordinatorJobBean coord = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0);
    _modifyCoordForFailureAction(coord, "wf-invalid-fork.xml");
    new CoordMaterializeTransitionXCommand(coord.getId(), 3600).call();
    final CoordJobGetJPAExecutor readCmd1 = new CoordJobGetJPAExecutor(coord.getId());
    waitFor(1 * 100, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            CoordinatorJobBean bean = jpaService.execute(readCmd1);
            return bean.getStatus() == CoordinatorJob.Status.SUCCEEDED || bean.getStatus() == CoordinatorJob.Status.KILLED;
        }
    });
    assertEquals(2, queue.size());
    assertEquals(EventStatus.WAITING, ((JobEvent) queue.poll()).getEventStatus());
    assertEquals(EventStatus.FAILURE, ((JobEvent) queue.poll()).getEventStatus());
    queue.clear();
}

Example 9 with CoordMaterializeTransitionXCommand

use of org.apache.oozie.command.coord.CoordMaterializeTransitionXCommand in project oozie by apache.

the class TestEventGeneration method testCoordinatorActionEvent.

@Test
public void testCoordinatorActionEvent() throws Exception {
    // avoid noise from other apptype events by setting it to only
    // coord action
    ehs.setAppTypes(new HashSet<String>(Arrays.asList("coordinator_action")));
    assertEquals(queue.size(), 0);
    Date startTime = DateUtils.parseDateOozieTZ("2013-01-01T10:00Z");
    Date endTime = DateUtils.parseDateOozieTZ("2013-01-01T10:01Z");
    CoordinatorJobBean coord = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0);
    modifyCoordForRunning(coord);
    // Action WAITING on materialization
    new CoordMaterializeTransitionXCommand(coord.getId(), 3600).call();
    final CoordActionGetJPAExecutor coordGetCmd = new CoordActionGetJPAExecutor(coord.getId() + "@1");
    CoordinatorActionBean action = jpaService.execute(coordGetCmd);
    assertEquals(CoordinatorAction.Status.WAITING, action.getStatus());
    assertEquals(1, queue.size());
    JobEvent event = (JobEvent) queue.poll();
    assertNotNull(event);
    assertEquals(EventStatus.WAITING, event.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, event.getAppType());
    assertEquals(action.getId(), event.getId());
    assertEquals(action.getJobId(), event.getParentId());
    assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime());
    assertNull(event.getStartTime());
    assertEquals(coord.getUser(), event.getUser());
    assertEquals(coord.getAppName(), event.getAppName());
    assertEquals(0, queue.size());
    // Make Action ready
    // In this case it will proceed to Running since n(ready_actions) < concurrency
    new CoordActionInputCheckXCommand(action.getId(), coord.getId()).call();
    action = jpaService.execute(coordGetCmd);
    assertEquals(CoordinatorAction.Status.RUNNING, action.getStatus());
    event = (JobEvent) queue.poll();
    assertEquals(EventStatus.STARTED, event.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, event.getAppType());
    assertEquals(action.getId(), event.getId());
    assertEquals(action.getJobId(), event.getParentId());
    assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime());
    WorkflowJobBean wfJob = jpaService.execute(new WorkflowJobGetJPAExecutor(action.getExternalId()));
    assertEquals(wfJob.getStartTime(), event.getStartTime());
    assertEquals(coord.getUser(), event.getUser());
    assertEquals(coord.getAppName(), event.getAppName());
    sleep(2000);
    // Action Success
    wfJob.setStatus(WorkflowJob.Status.SUCCEEDED);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfJob);
    action.setStatus(CoordinatorAction.Status.RUNNING);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    new CoordActionCheckXCommand(action.getId(), 0).call();
    action = jpaService.execute(coordGetCmd);
    assertEquals(CoordinatorAction.Status.SUCCEEDED, action.getStatus());
    List<Event> list = queue.pollBatch();
    event = (JobEvent) list.get(list.size() - 1);
    assertEquals(EventStatus.SUCCESS, event.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, event.getAppType());
    assertEquals(action.getId(), event.getId());
    assertEquals(action.getJobId(), event.getParentId());
    assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime());
    assertEquals(wfJob.getStartTime(), event.getStartTime());
    assertEquals(coord.getUser(), event.getUser());
    assertEquals(coord.getAppName(), event.getAppName());
    // Action Failure
    wfJob.setStatus(WorkflowJob.Status.FAILED);
    action.setStatus(CoordinatorAction.Status.RUNNING);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfJob);
    new CoordActionCheckXCommand(action.getId(), 0).call();
    action = jpaService.execute(coordGetCmd);
    assertEquals(CoordinatorAction.Status.FAILED, action.getStatus());
    event = (JobEvent) queue.poll();
    assertEquals(EventStatus.FAILURE, event.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, event.getAppType());
    assertEquals(action.getId(), event.getId());
    assertEquals(action.getJobId(), event.getParentId());
    assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime());
    assertEquals(wfJob.getStartTime(), event.getStartTime());
    assertEquals(coord.getUser(), event.getUser());
    assertEquals(coord.getAppName(), event.getAppName());
    // Action Suspended
    wfJob.setStatus(WorkflowJob.Status.SUSPENDED);
    action.setStatus(CoordinatorAction.Status.RUNNING);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfJob);
    new CoordActionCheckXCommand(action.getId(), 0).call();
    action = jpaService.execute(coordGetCmd);
    assertEquals(CoordinatorAction.Status.SUSPENDED, action.getStatus());
    event = (JobEvent) queue.poll();
    assertEquals(EventStatus.SUSPEND, event.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, event.getAppType());
    assertEquals(action.getId(), event.getId());
    assertEquals(action.getJobId(), event.getParentId());
    assertEquals(action.getNominalTime(), ((CoordinatorActionEvent) event).getNominalTime());
    assertEquals(wfJob.getStartTime(), event.getStartTime());
    assertEquals(coord.getUser(), event.getUser());
    assertEquals(coord.getAppName(), event.getAppName());
    // Action start on Coord Resume
    coord.setStatus(CoordinatorJobBean.Status.SUSPENDED);
    CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB_STATUS, coord);
    action.setStatus(CoordinatorAction.Status.SUSPENDED);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    wfJob.setStatus(WorkflowJob.Status.SUSPENDED);
    WorkflowInstance wfInstance = wfJob.getWorkflowInstance();
    ((LiteWorkflowInstance) wfInstance).setStatus(WorkflowInstance.Status.SUSPENDED);
    wfJob.setWorkflowInstance(wfInstance);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MODIFIED, wfJob);
    queue.clear();
    new CoordResumeXCommand(coord.getId()).call();
    waitForEventGeneration(1000);
    CoordinatorActionEvent cevent = (CoordinatorActionEvent) queue.poll();
    assertEquals(EventStatus.STARTED, cevent.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, cevent.getAppType());
    assertEquals(action.getId(), cevent.getId());
    assertEquals(action.getJobId(), cevent.getParentId());
    assertEquals(action.getNominalTime(), cevent.getNominalTime());
    coord = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, coord.getId());
    assertEquals(coord.getLastModifiedTime(), cevent.getStartTime());
    // Action going to WAITING on Coord Rerun
    action.setStatus(CoordinatorAction.Status.KILLED);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    queue.clear();
    new CoordRerunXCommand(coord.getId(), RestConstants.JOB_COORD_SCOPE_ACTION, "1", false, true, false, null).call();
    waitFor(3 * 100, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return jpaService.execute(coordGetCmd).getStatus() == CoordinatorAction.Status.WAITING;
        }
    });
    cevent = (CoordinatorActionEvent) queue.poll();
    assertEquals(EventStatus.WAITING, cevent.getEventStatus());
    assertEquals(AppType.COORDINATOR_ACTION, cevent.getAppType());
    assertEquals(action.getId(), cevent.getId());
    assertEquals(action.getJobId(), cevent.getParentId());
    assertEquals(action.getNominalTime(), cevent.getNominalTime());
    assertEquals(wfJob.getStartTime(), event.getStartTime());
    assertNotNull(cevent.getMissingDeps());
}

Example 10 with CoordMaterializeTransitionXCommand

use of org.apache.oozie.command.coord.CoordMaterializeTransitionXCommand in project oozie by apache.

the class TestCoordInputLogicPush method testHcatHdfs.

public void testHcatHdfs() throws Exception {
    Configuration conf = getConfForCombine();
    conf.set("initial_instance_a", "2014-10-07T00:00Z");
    conf.set("initial_instance_b", "2014-10-07T00:00Z");
    String inputLogic = // @formatter:off
    "<and name=\"test\">" + "<data-in name=\"testA\" dataset=\"A\" />" + "<data-in name=\"testB\" dataset=\"B\" />" + "</and>";
    // @formatter:on
    String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE);
    String input1 = createTestCaseSubDir("input-data/b/2014/10/08/_SUCCESS".split("/"));
    String input2 = addPartition("db_a", "table1", "dt=20141008;country=usa");
    new CoordMaterializeTransitionXCommand(jobId, 3600).call();
    new CoordPushDependencyCheckXCommand(jobId + "@1").call();
    new CoordActionInputCheckXCommand(jobId + "@1", jobId).call();
    startCoordAction(jobId);
    CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get(CoordActionQuery.GET_COORD_ACTION, jobId + "@1");
    assertFalse(CoordinatorAction.Status.WAITING.equals(actionBean.getStatus()));
    XConfiguration runConf = new XConfiguration(new StringReader(actionBean.getRunConf()));
    String dataSets = runConf.get("inputLogicData");
    assertEquals(dataSets.split(",").length, 2);
    checkDataSets(dataSets, input1, input2);
}