Examples with JPAService org.apache.oozie.service.JPAService used on opensource projects

Example 16 with JPAService

use of org.apache.oozie.service.JPAService in project oozie by apache.

the class PurgeXCommand method loadState.

/* (non-Javadoc)
     * @see org.apache.oozie.command.XCommand#loadState()
     */
@Override
protected void loadState() throws CommandException {
    try {
        jpaService = Services.get().get(JPAService.class);
        if (jpaService != null) {
            // Get the lists of workflows, coordinators, and bundles that can be purged (and have no parents)
            int size;
            do {
                size = wfList.size();
                wfList.addAll(jpaService.execute(new WorkflowJobsGetForPurgeJPAExecutor(wfOlderThan, wfList.size(), limit)));
            } while (size != wfList.size());
            if (purgeOldCoordAction) {
                LOG.debug("Purging workflows of long running coordinators is turned on");
                do {
                    size = coordActionList.size();
                    long olderThan = wfOlderThan;
                    List<WorkflowJobBean> jobBeans = WorkflowJobQueryExecutor.getInstance().getList(WorkflowJobQuery.GET_COMPLETED_COORD_WORKFLOWS_OLDER_THAN, olderThan, coordActionList.size(), limit);
                    for (WorkflowJobBean bean : jobBeans) {
                        coordActionList.add(bean.getParentId());
                        wfList.add(bean.getId());
                    }
                } while (size != coordActionList.size());
            }
            do {
                size = coordList.size();
                coordList.addAll(jpaService.execute(new CoordJobsGetForPurgeJPAExecutor(coordOlderThan, coordList.size(), limit)));
            } while (size != coordList.size());
            do {
                size = bundleList.size();
                bundleList.addAll(jpaService.execute(new BundleJobsGetForPurgeJPAExecutor(bundleOlderThan, bundleList.size(), limit)));
            } while (size != bundleList.size());
        } else {
            throw new CommandException(ErrorCode.E0610);
        }
    } catch (XException ex) {
        throw new CommandException(ex);
    }
}

Example 17 with JPAService

use of org.apache.oozie.service.JPAService in project oozie by apache.

the class TestActionCheckXCommand method _testActionCheckError.

private WorkflowActionBean _testActionCheckError() throws Exception {
    services.get(ActionService.class).registerAndInitExecutor(ErrorCheckActionExecutor.class);
    JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING);
    WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(action.getId());
    new ActionCheckXCommand(action.getId()).call();
    action = jpaService.execute(wfActionGetCmd);
    return action;
}

Example 18 with JPAService

use of org.apache.oozie.service.JPAService in project oozie by apache.

the class TestActionCheckXCommand method testActionCheckTransientDuringMRAction.

public void testActionCheckTransientDuringMRAction() throws Exception {
    // When using YARN, skip this test because it relies on shutting down the job tracker, which isn't used in YARN
    if (createJobConf().get("yarn.resourcemanager.address") != null) {
        return;
    }
    services.destroy();
    // Make the max number of retries lower so the test won't take as long
    final int maxRetries = 2;
    setSystemProperty("oozie.action.retries.max", Integer.toString(maxRetries));
    services = new Services();
    // Disable ActionCheckerService so it doesn't interfere by triggering any extra ActionCheckXCommands
    setClassesToBeExcluded(services.getConf(), new String[] { "org.apache.oozie.service.ActionCheckerService" });
    services.init();
    final JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job0 = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    final String jobId = job0.getId();
    WorkflowActionBean action0 = this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.PREP);
    final String actionId = action0.getId();
    final WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(actionId);
    new ActionStartXCommand(actionId, "map-reduce").call();
    final WorkflowActionBean action1 = jpaService.execute(wfActionGetCmd);
    String originalLauncherId = action1.getExternalId();
    ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job0, action1, false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
    String user = conf.get("user.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob launcherJob = jobClient.getJob(JobID.forName(originalLauncherId));
    waitFor(120 * 1000, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherHelper.hasIdSwap(actionData));
    new ActionCheckXCommand(action1.getId()).call();
    WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd);
    String originalMapperId = action2.getExternalChildIDs();
    assertFalse(originalLauncherId.equals(originalMapperId));
    // At this point, the launcher job has finished and the map-reduce action has started (but not finished)
    // Now, shutdown the job tracker to pretend it has gone down during the map-reduce job
    executeWhileJobTrackerIsShutdown(new ShutdownJobTrackerExecutable() {

        @Override
        public void execute() throws Exception {
            assertEquals(0, action1.getRetries());
            new ActionCheckXCommand(actionId).call();
            waitFor(30 * 1000, new Predicate() {

                @Override
                public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() > 0);
                }
            });
            waitFor(180 * 1000, new Predicate() {

                @Override
                public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() == 0);
                }
            });
            WorkflowActionBean action1b = jpaService.execute(wfActionGetCmd);
            assertEquals(0, action1b.getRetries());
            assertEquals("START_MANUAL", action1b.getStatusStr());
            WorkflowJobBean job1 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
            assertEquals("SUSPENDED", job1.getStatusStr());
        // At this point, the action has gotten a transient error, even after maxRetries tries so the workflow has been
        // SUSPENDED
        }
    });
    // Now, lets bring the job tracker back up and resume the workflow (which will restart the current action)
    // It should now continue and finish with SUCCEEDED
    new ResumeXCommand(jobId).call();
    WorkflowJobBean job2 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    assertEquals("RUNNING", job2.getStatusStr());
    sleep(500);
    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action3 = jpaService.execute(wfActionGetCmd);
    String launcherId = action3.getExternalId();
    assertFalse(originalLauncherId.equals(launcherId));
    final RunningJob launcherJob2 = jobClient.getJob(JobID.forName(launcherId));
    waitFor(120 * 1000, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return launcherJob2.isComplete();
        }
    });
    assertTrue(launcherJob2.isSuccessful());
    actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherHelper.hasIdSwap(actionData));
    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action4 = jpaService.execute(wfActionGetCmd);
    String mapperId = action4.getExternalChildIDs();
    assertFalse(originalMapperId.equals(mapperId));
    final RunningJob mrJob = jobClient.getJob(JobID.forName(mapperId));
    waitFor(120 * 1000, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action5 = jpaService.execute(wfActionGetCmd);
    assertEquals("SUCCEEDED", action5.getExternalStatus());
}

Example 19 with JPAService

use of org.apache.oozie.service.JPAService in project oozie by apache.

the class TestActionErrors method _testKillNodeErrorMessage.

private WorkflowActionBean _testKillNodeErrorMessage(String workflowXmlFile) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader(workflowXmlFile, -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", "end.error");
    conf.set("external-status", "FAILED/KILLED");
    conf.set("signal-value", "fail");
    final String jobId = engine.submitJob(conf, true);
    final JPAService jpaService = Services.get().get(JPAService.class);
    final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);
    waitFor(50000, new Predicate() {

        public boolean evaluate() throws Exception {
            WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
            return (job.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED);
        }
    });
    WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
    assertEquals(WorkflowJob.Status.KILLED, job.getStatus());
    WorkflowActionsGetForJobJPAExecutor wfActionsGetCmd = new WorkflowActionsGetForJobJPAExecutor(jobId);
    List<WorkflowActionBean> actions = jpaService.execute(wfActionsGetCmd);
    WorkflowActionBean action = null;
    WorkflowActionBean killAction = null;
    for (WorkflowActionBean bean : actions) {
        if (bean.getType().equals("test")) {
            action = bean;
        }
        if (bean.getType().equals(KillActionExecutor.TYPE)) {
            killAction = bean;
        }
    }
    assertNotNull(action);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals("end", action.getErrorMessage());
    assertEquals(WorkflowAction.Status.ERROR, action.getStatus());
    assertNotNull(killAction);
    return killAction;
}

Example 20 with JPAService

use of org.apache.oozie.service.JPAService in project oozie by apache.

the class TestActionErrors method _testErrorWithUserRetry.

/**
 * Provides functionality to test user retry
 *
 * @param errorType the error type. (start.non-transient, end.non-transient)
 * @param externalStatus the external status to set.
 * @param signalValue the signal value to set.
 * @throws Exception
 */
private void _testErrorWithUserRetry(String errorType, String externalStatus, String signalValue) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid-user-retry.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", errorType);
    conf.set("external-status", externalStatus);
    conf.set("signal-value", signalValue);
    final String jobId = engine.submitJob(conf, true);
    final JPAService jpaService = Services.get().get(JPAService.class);
    final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);
    final WorkflowActionsGetForJobJPAExecutor actionsGetExecutor = new WorkflowActionsGetForJobJPAExecutor(jobId);
    waitFor(5000, new Predicate() {

        public boolean evaluate() throws Exception {
            List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor);
            WorkflowActionBean action = null;
            for (WorkflowActionBean bean : actions) {
                if (bean.getType().equals("test")) {
                    action = bean;
                    break;
                }
            }
            return (action != null && action.getUserRetryCount() == 2);
        }
    });
    List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor);
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
        if (bean.getType().equals("test")) {
            action = bean;
            break;
        }
    }
    assertNotNull(action);
    assertEquals(2, action.getUserRetryCount());
}