Examples with CoordinatorActionBean org.apache.oozie.CoordinatorActionBean used on opensource projects

Example 11 with CoordinatorActionBean

use of org.apache.oozie.CoordinatorActionBean in project oozie by apache.

the class TestStatusTransitService method testCoordStatusTransitRunningFromKilled.

// Test coord transition from killed to running when one action is rerun.
public void testCoordStatusTransitRunningFromKilled() throws Exception {
    final JPAService jpaService = Services.get().get(JPAService.class);
    String currentDatePlusMonth = XDataTestCase.getCurrentDateafterIncrementingInMonths(1);
    Date start = DateUtils.parseDateOozieTZ(currentDatePlusMonth);
    Date end = DateUtils.parseDateOozieTZ(currentDatePlusMonth);
    CoordinatorJobBean coordJob = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, start, end, false, false, 1);
    final CoordinatorActionBean coordAction = addRecordToCoordActionTable(coordJob.getId(), 1, CoordinatorAction.Status.RUNNING, "coord-action-get.xml", null, "RUNNING", 0);
    new CoordKillXCommand(coordJob.getId()).call();
    final CoordJobGetJPAExecutor coordJobGetCmd = new CoordJobGetJPAExecutor(coordJob.getId());
    waitFor(5 * 1000, new Predicate() {

        public boolean evaluate() throws Exception {
            CoordinatorJobBean coordJob = jpaService.execute(coordJobGetCmd);
            return coordJob.getStatusStr().equals("KILLED");
        }
    });
    Runnable runnable = new StatusTransitRunnable();
    runnable.run();
    sleep(1000);
    coordJob = jpaService.execute(coordJobGetCmd);
    assertEquals(CoordinatorJob.Status.KILLED, coordJob.getStatus());
    coordAction.setStatus(CoordinatorAction.Status.RUNNING);
    coordJob.setPending();
    CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB_STATUS_PENDING, coordJob);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION, coordAction);
    runnable.run();
    sleep(1000);
    coordJob = jpaService.execute(coordJobGetCmd);
    assertEquals(CoordinatorJob.Status.RUNNING, coordJob.getStatus());
}

Example 12 with CoordinatorActionBean

use of org.apache.oozie.CoordinatorActionBean in project oozie by apache.

the class TestHAPartitionDependencyManagerService method testPurgeMissingDependenciesForCache.

protected void testPurgeMissingDependenciesForCache(PartitionDependencyManagerService pdms) throws Exception {
    String actionId1 = "1234465451";
    String actionId2 = "1234465452";
    String actionId3 = "1234465453";
    // add partitions as missing
    HCatURI dep1 = new HCatURI("hcat://hcat-server1.domain.com:5080/mydb/mytbl1/dt=20120101;country=us");
    HCatURI dep2 = new HCatURI("hcat://hcat-server1.domain.com:5080/mydb/mytbl1/country=us;dt=20120101");
    HCatURI dep3 = new HCatURI("hcat://hcat-server2.domain.com:5080/mydb/mytbl2/dt=20120102;country=us");
    // actionId1-->(dep1,2), actionId2-->(dep2), actionId3-->(dep2,3)
    addMissingDependencyAndRegister(dep1, actionId1, pdms);
    addMissingDependencyAndRegister(dep2, actionId1, pdms);
    addMissingDependencyAndRegister(dep2, actionId2, pdms);
    addMissingDependencyAndRegister(dep2, actionId3, pdms);
    addMissingDependencyAndRegister(dep3, actionId3, pdms);
    List<String> waitingDep1 = (ArrayList<String>) pdms.getWaitingActions(dep1);
    assertEquals(waitingDep1.size(), 1);
    assertEquals(waitingDep1.get(0), actionId1);
    List<String> waitingDep2 = (ArrayList<String>) pdms.getWaitingActions(dep2);
    assertEquals(waitingDep2.size(), 3);
    for (String id : waitingDep2) {
        assertTrue(id.equals(actionId1) || id.equals(actionId2) || id.equals(actionId3));
    }
    List<String> waitingDep3 = (ArrayList<String>) pdms.getWaitingActions(dep3);
    assertEquals(waitingDep3.size(), 1);
    assertTrue(waitingDep3.get(0).equals(actionId3));
    // make only coordAction 1 to WAITING, the rest to RUNNING (only WAITING
    // remain dependency cache)
    ArrayList<JsonBean> insertList = new ArrayList<JsonBean>();
    CoordinatorActionBean coordAction1 = new CoordinatorActionBean();
    coordAction1.setId(actionId1);
    coordAction1.setStatus(Status.WAITING);
    insertList.add(coordAction1);
    CoordinatorActionBean coordAction2 = new CoordinatorActionBean();
    coordAction2.setId(actionId2);
    coordAction2.setStatus(Status.RUNNING);
    insertList.add(coordAction2);
    CoordinatorActionBean coordAction3 = new CoordinatorActionBean();
    coordAction3.setId(actionId3);
    coordAction3.setStatus(Status.RUNNING);
    insertList.add(coordAction3);
    BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, null, null);
    // run cache purge
    Services.get().getConf().setInt(PartitionDependencyManagerService.CACHE_PURGE_TTL, 0);
    pdms.runCachePurgeWorker();
    // only coord Action 1 still in dependency cache
    waitingDep1 = (ArrayList<String>) pdms.getWaitingActions(dep1);
    assertEquals(waitingDep1.size(), 1);
    assertTrue(waitingDep1.get(0).equals(actionId1));
    // only coord Action 1 still in dependency cache
    waitingDep2 = (ArrayList<String>) pdms.getWaitingActions(dep2);
    assertEquals(waitingDep2.size(), 1);
    assertTrue(waitingDep2.get(0).equals(actionId1));
    waitingDep3 = (ArrayList<String>) pdms.getWaitingActions(dep3);
    assertNull(waitingDep3);
    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
    // mytbl1 should be still in topic map
    assertTrue(hcatService.isRegisteredForNotification(dep1));
    // mytbl1 should be still in topic map
    assertTrue(hcatService.isRegisteredForNotification(dep2));
    // mytbl2 should NOT be in topic map
    assertFalse(hcatService.isRegisteredForNotification(dep3));
}

Example 13 with CoordinatorActionBean

use of org.apache.oozie.CoordinatorActionBean in project oozie by apache.

the class TestJMSTopicService method testMixedTopic1.

@Test
public void testMixedTopic1() throws Exception {
    services = setupServicesForTopic();
    services.getConf().set(JMSTopicService.TOPIC_NAME, JMSTopicService.JobType.WORKFLOW.getValue() + " = workflow," + JMSTopicService.JobType.COORDINATOR.getValue() + "=coord, default = " + JMSTopicService.TopicType.JOBID.getValue());
    services.init();
    JMSTopicService jmsTopicService = Services.get().get(JMSTopicService.class);
    WorkflowJobBean wfj = addRecordToWfJobTable(WorkflowJob.Status.SUCCEEDED, WorkflowInstance.Status.SUCCEEDED);
    assertEquals("workflow", jmsTopicService.getTopic(wfj.getId()));
    assertEquals("workflow", jmsTopicService.getTopic(AppType.WORKFLOW_JOB, wfj.getUser(), wfj.getId(), null));
    WorkflowActionBean wab = addRecordToWfActionTable(wfj.getId(), "1", WorkflowAction.Status.RUNNING);
    assertEquals("workflow", jmsTopicService.getTopic(wab.getId()));
    assertEquals("workflow", jmsTopicService.getTopic(AppType.WORKFLOW_ACTION, wfj.getUser(), wab.getId(), wab.getWfId()));
    CoordinatorJobBean cjb = addRecordToCoordJobTable(CoordinatorJob.Status.SUCCEEDED, true, true);
    assertEquals("coord", jmsTopicService.getTopic(cjb.getId()));
    assertEquals("coord", jmsTopicService.getTopic(AppType.COORDINATOR_JOB, cjb.getUser(), cjb.getId(), null));
    CoordinatorActionBean cab = addRecordToCoordActionTable(cjb.getId(), 1, CoordinatorAction.Status.SUCCEEDED, "coord-action-for-action-input-check.xml", 0);
    assertEquals("coord", jmsTopicService.getTopic(cab.getId()));
    assertEquals("coord", jmsTopicService.getTopic(AppType.COORDINATOR_ACTION, cjb.getUser(), cab.getId(), cab.getJobId()));
    BundleJobBean bjb = addRecordToBundleJobTable(Job.Status.RUNNING, true);
    assertEquals(bjb.getId(), jmsTopicService.getTopic(bjb.getId()));
    assertEquals(bjb.getId(), jmsTopicService.getTopic(AppType.BUNDLE_JOB, bjb.getUser(), bjb.getId(), null));
    BundleActionBean bab = addRecordToBundleActionTable(bjb.getId(), "1", 1, Job.Status.RUNNING);
    assertEquals(bjb.getId(), jmsTopicService.getTopic(bab.getBundleActionId()));
    assertEquals(bjb.getId(), jmsTopicService.getTopic(AppType.BUNDLE_ACTION, bjb.getUser(), bab.getBundleActionId(), bab.getBundleId()));
}

Example 14 with CoordinatorActionBean

use of org.apache.oozie.CoordinatorActionBean in project oozie by apache.

the class TestJMSTopicService method testMixedTopic2.

@Test
public void testMixedTopic2() throws Exception {
    services = setupServicesForTopic();
    services.getConf().set(JMSTopicService.TOPIC_NAME, JMSTopicService.JobType.WORKFLOW.getValue() + " = workflow," + JMSTopicService.JobType.COORDINATOR.getValue() + "=coord");
    services.init();
    JMSTopicService jmsTopicService = Services.get().get(JMSTopicService.class);
    WorkflowJobBean wfj = addRecordToWfJobTable(WorkflowJob.Status.SUCCEEDED, WorkflowInstance.Status.SUCCEEDED);
    assertEquals("workflow", jmsTopicService.getTopic(wfj.getId()));
    assertEquals("workflow", jmsTopicService.getTopic(AppType.WORKFLOW_JOB, wfj.getUser(), wfj.getId(), null));
    WorkflowActionBean wab = addRecordToWfActionTable(wfj.getId(), "1", WorkflowAction.Status.RUNNING);
    assertEquals("workflow", jmsTopicService.getTopic(wab.getId()));
    assertEquals("workflow", jmsTopicService.getTopic(AppType.WORKFLOW_ACTION, wfj.getUser(), wab.getId(), wab.getWfId()));
    CoordinatorJobBean cjb = addRecordToCoordJobTable(CoordinatorJob.Status.SUCCEEDED, true, true);
    assertEquals("coord", jmsTopicService.getTopic(cjb.getId()));
    CoordinatorActionBean cab = addRecordToCoordActionTable(cjb.getId(), 1, CoordinatorAction.Status.SUCCEEDED, "coord-action-for-action-input-check.xml", 0);
    assertEquals("coord", jmsTopicService.getTopic(cab.getId()));
    assertEquals("coord", jmsTopicService.getTopic(AppType.COORDINATOR_ACTION, cjb.getUser(), cab.getId(), cab.getJobId()));
    BundleJobBean bjb = addRecordToBundleJobTable(Job.Status.RUNNING, true);
    // As no default is specified, user will be considered as topic
    assertEquals(bjb.getUser(), jmsTopicService.getTopic(bjb.getId()));
    assertEquals(bjb.getUser(), jmsTopicService.getTopic(AppType.BUNDLE_JOB, bjb.getUser(), bjb.getId(), null));
    BundleActionBean bab = addRecordToBundleActionTable(bjb.getId(), "1", 1, Job.Status.RUNNING);
    assertEquals(bjb.getUser(), jmsTopicService.getTopic(bab.getBundleActionId()));
    assertEquals(bjb.getUser(), jmsTopicService.getTopic(AppType.BUNDLE_ACTION, bjb.getUser(), bab.getBundleActionId(), bab.getBundleId()));
}

Example 15 with CoordinatorActionBean

use of org.apache.oozie.CoordinatorActionBean in project oozie by apache.

the class CoordActionReadyXCommand method execute.

@Override
protected /**
 * Check for READY actions and change state to SUBMITTED by a command to submit the job to WF engine.
 * This method checks all the actions associated with a jobId to figure out which actions
 * to start (based on concurrency and execution order [FIFO, LIFO, LAST_ONLY, NONE])
 */
Void execute() throws CommandException {
    // number of actions to start (-1 means start ALL)
    int numActionsToStart = -1;
    // get execution setting for this job (FIFO, LIFO, LAST_ONLY)
    CoordinatorJob.Execution jobExecution = coordJob.getExecutionOrder();
    // get concurrency setting for this job
    int jobConcurrency = coordJob.getConcurrency();
    // if less than 0, then UNLIMITED concurrency
    if (jobConcurrency >= 0) {
        // count number of actions that are already RUNNING or SUBMITTED
        // subtract from CONCURRENCY to calculate number of actions to start
        // in WF engine
        int numRunningJobs;
        try {
            numRunningJobs = jpaService.execute(new CoordJobGetRunningActionsCountJPAExecutor(jobId));
        } catch (JPAExecutorException je) {
            throw new CommandException(je);
        }
        numActionsToStart = jobConcurrency - numRunningJobs;
        if (numActionsToStart < 0) {
            numActionsToStart = 0;
        }
        log.debug("concurrency=" + jobConcurrency + ", execution=" + jobExecution + ", numRunningJobs=" + numRunningJobs + ", numLeftover=" + numActionsToStart);
        // no actions to start
        if (numActionsToStart == 0) {
            log.info("Not starting any additional actions because max concurrency [{0}]" + " for coordinator [{1}] has been reached.", jobConcurrency, jobId);
        }
    }
    // get list of actions that are READY and fit in the concurrency and execution
    List<CoordinatorActionBean> actions;
    try {
        actions = jpaService.execute(new CoordJobGetReadyActionsJPAExecutor(jobId, jobExecution.name()));
    } catch (JPAExecutorException je) {
        throw new CommandException(je);
    }
    log.debug("Number of READY actions = " + actions.size());
    Date now = new Date();
    // If we're using LAST_ONLY or NONE, we should check if any of these need to be SKIPPED instead of SUBMITTED
    if (jobExecution.equals(CoordinatorJobBean.Execution.LAST_ONLY)) {
        for (Iterator<CoordinatorActionBean> it = actions.iterator(); it.hasNext(); ) {
            CoordinatorActionBean action = it.next();
            try {
                Date nextNominalTime = CoordCommandUtils.computeNextNominalTime(coordJob, action);
                if (nextNominalTime != null) {
                    // action should be started; so set it to SKIPPED
                    if (now.after(nextNominalTime)) {
                        LOG.info("LAST_ONLY execution: Preparing to skip action [{0}] because the current time [{1}] is later " + "than the nominal time [{2}] of the next action]", action.getId(), DateUtils.formatDateOozieTZ(now), DateUtils.formatDateOozieTZ(nextNominalTime));
                        queue(new CoordActionSkipXCommand(action, coordJob.getUser(), coordJob.getAppName()));
                        it.remove();
                    } else {
                        LOG.debug("LAST_ONLY execution: Not skipping action [{0}] because the current time [{1}] is earlier " + "than the nominal time [{2}] of the next action]", action.getId(), DateUtils.formatDateOozieTZ(now), DateUtils.formatDateOozieTZ(nextNominalTime));
                    }
                }
            } catch (ParseException e) {
                LOG.error("Should not happen", e);
            } catch (JDOMException e) {
                LOG.error("Should not happen", e);
            }
        }
    } else if (jobExecution.equals(CoordinatorJobBean.Execution.NONE)) {
        for (Iterator<CoordinatorActionBean> it = actions.iterator(); it.hasNext(); ) {
            CoordinatorActionBean action = it.next();
            // If the current time is after the nominal time of this action plus some tolerance,
            // then we've passed the window where this action should be started; so set it to SKIPPED
            Calendar cal = Calendar.getInstance(DateUtils.getTimeZone(coordJob.getTimeZone()));
            cal.setTime(action.getNominalTime());
            int tolerance = ConfigurationService.getInt(CoordActionInputCheckXCommand.COORD_EXECUTION_NONE_TOLERANCE);
            cal.add(Calendar.MINUTE, tolerance);
            if (now.after(cal.getTime())) {
                LOG.info("NONE execution: Preparing to skip action [{0}] because the current time [{1}] is more than [{2}]" + " minutes later than the nominal time [{3}] of the current action]", action.getId(), DateUtils.formatDateOozieTZ(now), tolerance, DateUtils.formatDateOozieTZ(action.getNominalTime()));
                queue(new CoordActionSkipXCommand(action, coordJob.getUser(), coordJob.getAppName()));
                it.remove();
            } else {
                LOG.debug("NONE execution: Not skipping action [{0}] because the current time [{1}] is earlier than [{2}]" + " minutes later than the nominal time [{3}] of the current action]", action.getId(), DateUtils.formatDateOozieTZ(now), tolerance, DateUtils.formatDateOozieTZ(action.getNominalTime()));
            }
        }
    }
    int counter = 0;
    for (CoordinatorActionBean action : actions) {
        // actions), or if the counter is less than numActionsToStart
        if ((numActionsToStart < 0) || (counter < numActionsToStart)) {
            log.debug("Set status to SUBMITTED for id: " + action.getId());
            // change state of action to SUBMITTED
            action.setStatus(CoordinatorAction.Status.SUBMITTED);
            try {
                CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
            } catch (JPAExecutorException je) {
                throw new CommandException(je);
            }
            // start action
            new CoordActionStartXCommand(action.getId(), coordJob.getUser(), coordJob.getAppName(), action.getJobId()).call();
        } else {
            break;
        }
        counter++;
    }
    return null;
}