Examples with ActionExecutorContext org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext used on opensource projects

Example 6 with ActionExecutorContext

use of org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext in project oozie by apache.

the class WorkflowActionRetryInfoXCommand method execute.

@Override
protected List<Map<String, String>> execute() throws CommandException {
    List<Map<String, String>> retriesList = new ArrayList<Map<String, String>>();
    ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(wfJob, wfAction);
    for (int i = 0; i < wfAction.getUserRetryCount(); i++) {
        Map<String, String> retries = new HashMap<String, String>();
        String value = context.getVar(JobUtils.getRetryKey(JsonTags.WORKFLOW_ACTION_EXTERNAL_CHILD_IDS, i));
        if (value != null) {
            retries.put(JsonTags.WORKFLOW_ACTION_EXTERNAL_CHILD_IDS, value);
        }
        value = context.getVar(JobUtils.getRetryKey(JsonTags.WORKFLOW_ACTION_CONSOLE_URL, i));
        if (value != null) {
            retries.put(JsonTags.WORKFLOW_ACTION_CONSOLE_URL, value);
        }
        value = context.getVar(JobUtils.getRetryKey(JsonTags.WORKFLOW_ACTION_START_TIME, i));
        if (value != null) {
            retries.put(JsonTags.WORKFLOW_ACTION_START_TIME, JsonUtils.formatDateRfc822(new Date(Long.parseLong(value))));
        }
        value = context.getVar(JobUtils.getRetryKey(JsonTags.WORKFLOW_ACTION_END_TIME, i));
        if (value != null) {
            retries.put(JsonTags.WORKFLOW_ACTION_END_TIME, JsonUtils.formatDateRfc822(new Date(Long.parseLong(value))));
        }
        retries.put(JsonTags.ACTION_ATTEMPT, String.valueOf(i + 1));
        retriesList.add(retries);
    }
    return retriesList;
}

Example 7 with ActionExecutorContext

use of org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext in project oozie by apache.

the class TestOozieJobInfo method testInfoWithBundle.

public void testInfoWithBundle() throws Exception {
    Services.get().getConf().setBoolean(OozieJobInfo.CONF_JOB_INFO, true);
    OozieJobInfo.setJobInfo(true);
    BundleJobBean job = this.addRecordToBundleJobTable(Job.Status.PREP, false);
    final JPAService jpaService = Services.get().get(JPAService.class);
    Configuration jobConf = null;
    try {
        jobConf = new XConfiguration(new StringReader(job.getConf()));
    } catch (IOException ioe) {
        log.warn("Configuration parse error. read from DB :" + job.getConf(), ioe);
        throw new CommandException(ErrorCode.E1005, ioe);
    }
    setCoordConf(jobConf);
    Path appPath = new Path(jobConf.get(OozieClient.BUNDLE_APP_PATH), "bundle.xml");
    jobConf.set(OozieClient.BUNDLE_APP_PATH, appPath.toString());
    BundleSubmitXCommand submitCmd = new BundleSubmitXCommand(jobConf);
    submitCmd.call();
    BundleJobGetJPAExecutor bundleJobGetExecutor = new BundleJobGetJPAExecutor(submitCmd.getJob().getId());
    job = jpaService.execute(bundleJobGetExecutor);
    assertEquals(job.getStatus(), Job.Status.PREP);
    new BundleStartXCommand(job.getId()).call();
    sleep(2000);
    List<BundleActionBean> actions = BundleActionQueryExecutor.getInstance().getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, job.getId());
    assertEquals(1, actions.size());
    final String bundleID = job.getId();
    waitFor(200000, new Predicate() {

        public boolean evaluate() throws Exception {
            List<BundleActionBean> actions = BundleActionQueryExecutor.getInstance().getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, bundleID);
            return actions.get(0).getStatus().equals(Job.Status.RUNNING);
        }
    });
    actions = BundleActionQueryExecutor.getInstance().getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, job.getId());
    final String cordID = actions.get(0).getCoordId();
    waitFor(200000, new Predicate() {

        public boolean evaluate() throws Exception {
            CoordJobGetJPAExecutor coordGetCmd2 = new CoordJobGetJPAExecutor(cordID);
            CoordinatorJobBean cc = jpaService.execute(coordGetCmd2);
            return cc.getStatus().equals(Job.Status.RUNNING);
        }
    });
    final String jobID = jpaService.execute(new WorkflowJobsGetFromCoordParentIdJPAExecutor(cordID, 1)).get(0);
    final WorkflowActionsGetForJobJPAExecutor actionsGetExecutor = new WorkflowActionsGetForJobJPAExecutor(jobID);
    waitFor(200000, new Predicate() {

        public boolean evaluate() throws Exception {
            List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor);
            WorkflowActionBean action = null;
            for (WorkflowActionBean bean : actions) {
                if (bean.getName().contains("hadoop")) {
                    action = bean;
                    break;
                }
            }
            return action.getStatus().toString().equalsIgnoreCase(Job.Status.RUNNING.toString());
        }
    });
    final WorkflowJobGetJPAExecutor wfeExc = new WorkflowJobGetJPAExecutor(jobID);
    WorkflowJobBean wfbean = jpaService.execute(wfeExc);
    List<WorkflowActionBean> actionList = jpaService.execute(actionsGetExecutor);
    ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(wfbean, actionList.get(1), false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(actionList.get(1).getConf()));
    String user = conf.get("user.name");
    FileSystem fs = getFileSystem();
    Configuration jobXmlConf = new XConfiguration(fs.open(getPathToWorkflowResource(user, wfbean, services, context, LauncherAM.LAUNCHER_JOB_CONF_XML)));
    String jobInfo = jobXmlConf.get(OozieJobInfo.JOB_INFO_KEY);
    // BUNDLE_ID;BUNDLE_NAME;COORDINATOR_NAME;COORDINATOR_NOMINAL_TIME;
    // WORKFLOW_ID;WORKFLOW_NAME;WORKFLOW_DEPTH;WORKFLOW_SUPERPARENT;
    // ACTION_TYPE;ACTION_NAME,JOB_INFO,custom_info;
    assertEquals(jobInfo.split(OozieJobInfo.SEPARATOR).length, 13);
    assertTrue(jobInfo.contains(bundleID));
    assertTrue(jobInfo.contains("bundle.name=test_bundle,"));
    assertTrue(jobInfo.contains(cordID));
    assertTrue(jobInfo.contains("action.type=map-reduce"));
    assertTrue(jobInfo.contains("wf.depth=0"));
    assertTrue(jobInfo.contains("wf.superparent.id=" + cordID));
    assertTrue(jobInfo.contains(",testing=test,"));
    assertTrue(jobInfo.contains(",coord.nominal.time="));
    assertTrue(jobInfo.contains("launcher=true"));
}

Example 8 with ActionExecutorContext

use of org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext in project oozie by apache.

the class TestActionCheckXCommand method testActionCheck.

public void testActionCheck() throws Exception {
    JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);
    WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(action.getId());
    new ActionStartXCommand(action.getId(), "map-reduce").call();
    action = jpaService.execute(wfActionGetCmd);
    ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job, action, false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
    String user = conf.get("user.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    String launcherId = action.getExternalId();
    waitUntilYarnAppDoneAndAssertSuccess(launcherId);
    Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherHelper.hasIdSwap(actionData));
    new ActionCheckXCommand(action.getId()).call();
    action = jpaService.execute(wfActionGetCmd);
    String externalId = action.getExternalId();
    String childId = action.getExternalChildIDs();
    assertEquals("LauncherId", launcherId, externalId);
    assertNotNull(childId);
    waitUntilYarnAppDoneAndAssertSuccess(childId);
    new ActionCheckXCommand(action.getId()).call();
    action = jpaService.execute(wfActionGetCmd);
    assertEquals("SUCCEEDED", action.getExternalStatus());
}

Example 9 with ActionExecutorContext

use of org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext in project oozie by apache.

the class TestActionCheckXCommand method testActionCheckTransientDuringLauncher.

public void testActionCheckTransientDuringLauncher() throws Exception {
    // When using YARN, skip this test because it relies on shutting down the job tracker, which isn't used in YARN
    if (createJobConf().get("yarn.resourcemanager.address") != null) {
        return;
    }
    services.destroy();
    // Make the max number of retries lower so the test won't take as long
    final int maxRetries = 2;
    setSystemProperty("oozie.action.retries.max", Integer.toString(maxRetries));
    services = new Services();
    // Disable ActionCheckerService so it doesn't interfere by triggering any extra ActionCheckXCommands
    setClassesToBeExcluded(services.getConf(), new String[] { "org.apache.oozie.service.ActionCheckerService" });
    services.init();
    final JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job0 = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    final String jobId = job0.getId();
    WorkflowActionBean action0 = this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.PREP);
    final String actionId = action0.getId();
    final WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(actionId);
    new ActionStartXCommand(actionId, "map-reduce").call();
    final WorkflowActionBean action1 = jpaService.execute(wfActionGetCmd);
    String originalLauncherId = action1.getExternalId();
    // At this point, the launcher job has started (but not finished)
    // Now, shutdown the job tracker to pretend it has gone down during the launcher job
    executeWhileJobTrackerIsShutdown(new ShutdownJobTrackerExecutable() {

        @Override
        public void execute() throws Exception {
            assertEquals(0, action1.getRetries());
            new ActionCheckXCommand(actionId).call();
            waitFor(30 * 1000, new Predicate() {

                @Override
                public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() > 0);
                }
            });
            waitFor(180 * 1000, new Predicate() {

                @Override
                public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() == 0);
                }
            });
            WorkflowActionBean action1b = jpaService.execute(wfActionGetCmd);
            assertEquals(0, action1b.getRetries());
            assertEquals("START_MANUAL", action1b.getStatusStr());
            WorkflowJobBean job1 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
            assertEquals("SUSPENDED", job1.getStatusStr());
        // At this point, the action has gotten a transient error, even after maxRetries tries so the workflow has been
        // SUSPENDED
        }
    });
    // Now, lets bring the job tracker back up and resume the workflow (which will restart the current action)
    // It should now continue and finish with SUCCEEDED
    new ResumeXCommand(jobId).call();
    WorkflowJobBean job2 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    assertEquals("RUNNING", job2.getStatusStr());
    ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job2, action1, false, false);
    WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action2.getConf()));
    String user = conf.get("user.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action3 = jpaService.execute(wfActionGetCmd);
    String launcherId = action3.getExternalId();
    assertFalse(originalLauncherId.equals(launcherId));
    final RunningJob launcherJob = jobClient.getJob(JobID.forName(launcherId));
    waitFor(120 * 1000, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherHelper.hasIdSwap(actionData));
    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action4 = jpaService.execute(wfActionGetCmd);
    String mapperId = action4.getExternalId();
    String childId = action4.getExternalChildIDs();
    assertTrue(launcherId.equals(mapperId));
    final RunningJob mrJob = jobClient.getJob(JobID.forName(childId));
    waitFor(120 * 1000, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action5 = jpaService.execute(wfActionGetCmd);
    assertEquals("SUCCEEDED", action5.getExternalStatus());
}

Example 10 with ActionExecutorContext

use of org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext in project oozie by apache.

the class TestActionStartXCommand method testActionWithEscapedStringAndCDATA.

/**
 * Test workflow action with CDATA section and string configuration that needs to be escaped can be submitted without
 * throwing exceptions.
 * <p/>
 * Escaped string needs to be 'escaped' before converting to XML Document, otherwise,
 * exception will be thrown.
 * @see org.apache.oozie.DagELFunctions#configureEvaluator(org.apache.oozie.util.ELEvaluator.ELEvaluator evaluator,
 *  org.apache.oozie.WorkflowJobBean, org.apache.oozie.WorkflowActionBean)
 *
 * @throws Exception thrown if failed to execute test case
 */
public void testActionWithEscapedStringAndCDATA() throws Exception {
    // create workflow job and action beans with escaped parameters and CDATA value
    JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job = this.addRecordToWfJobTableWithEscapedStringAndCDATA(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean action = this.addRecordToWfActionTableWithEscapedStringAndCDATA(job.getId(), WorkflowAction.Status.PREP);
    WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(action.getId());
    // start workflow action
    new ActionStartXCommand(action.getId(), "map-reduce").call();
    action = jpaService.execute(wfActionGetCmd);
    assertNotNull(action.getExternalId());
    ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job, action, false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
    String user = conf.get("user.name");
    String launcherId = action.getExternalId();
    waitUntilYarnAppDoneAndAssertSuccess(launcherId);
    Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherHelper.hasIdSwap(actionData));
}